Use LevelDB storage backend

.gitignore

@@ -7,3 +7,4 @@ syncthing.exe
 discosrv
 .jshintrc
 coverage.out
+files/pidx

Godeps/Godeps.json

@@ -32,6 +32,11 @@
 			"Comment": "null-81",
 			"Rev": "9cbe983aed9b0dfc73954433fead5e00866342ac"
 		},
+		{
+			"ImportPath": "code.google.com/p/snappy-go/snappy",
+			"Comment": "null-15",
+			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
+		},
 		{
 			"ImportPath": "github.com/golang/groupcache/lru",
 			"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
@@ -40,6 +45,10 @@
 			"ImportPath": "github.com/juju/ratelimit",
 			"Rev": "cbaa435c80a9716e086f25d409344b26c4039358"
 		},
+		{
+			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
+			"Rev": "ca1565e5fb6658691d7074d270602c9185a55c79"
+		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
 			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go

@@ -0,0 +1,124 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+// ErrCorrupt reports that the input is invalid.
+var ErrCorrupt = errors.New("snappy: corrupt input")
+
+// DecodedLen returns the length of the decoded block.
+func DecodedLen(src []byte) (int, error) {
+	v, _, err := decodedLen(src)
+	return v, err
+}
+
+// decodedLen returns the length of the decoded block and the number of bytes
+// that the length header occupied.
+func decodedLen(src []byte) (blockLen, headerLen int, err error) {
+	v, n := binary.Uvarint(src)
+	if n == 0 {
+		return 0, 0, ErrCorrupt
+	}
+	if uint64(int(v)) != v {
+		return 0, 0, errors.New("snappy: decoded block is too large")
+	}
+	return int(v), n, nil
+}
+
+// Decode returns the decoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire decoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Decode(dst, src []byte) ([]byte, error) {
+	dLen, s, err := decodedLen(src)
+	if err != nil {
+		return nil, err
+	}
+	if len(dst) < dLen {
+		dst = make([]byte, dLen)
+	}
+
+	var d, offset, length int
+	for s < len(src) {
+		switch src[s] & 0x03 {
+		case tagLiteral:
+			x := uint(src[s] >> 2)
+			switch {
+			case x < 60:
+				s += 1
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
+	}
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
+}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/encode.go

@@ -0,0 +1,174 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+)
+
+// We limit how far copy back-references can go, the same as the C++ code.
+const maxOffset = 1 << 15
+
+// emitLiteral writes a literal chunk and returns the number of bytes written.
+func emitLiteral(dst, lit []byte) int {
+	i, n := 0, uint(len(lit)-1)
+	switch {
+	case n < 60:
+		dst[0] = uint8(n)<<2 | tagLiteral
+		i = 1
+	case n < 1<<8:
+		dst[0] = 60<<2 | tagLiteral
+		dst[1] = uint8(n)
+		i = 2
+	case n < 1<<16:
+		dst[0] = 61<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		i = 3
+	case n < 1<<24:
+		dst[0] = 62<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		i = 4
+	case int64(n) < 1<<32:
+		dst[0] = 63<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		dst[4] = uint8(n >> 24)
+		i = 5
+	default:
+		panic("snappy: source buffer is too long")
+	}
+	if copy(dst[i:], lit) != len(lit) {
+		panic("snappy: destination buffer is too short")
+	}
+	return i + len(lit)
+}
+
+// emitCopy writes a copy chunk and returns the number of bytes written.
+func emitCopy(dst []byte, offset, length int) int {
+	i := 0
+	for length > 0 {
+		x := length - 4
+		if 0 <= x && x < 1<<3 && offset < 1<<11 {
+			dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
+			dst[i+1] = uint8(offset)
+			i += 2
+			break
+		}
+
+		x = length
+		if x > 1<<6 {
+			x = 1 << 6
+		}
+		dst[i+0] = uint8(x-1)<<2 | tagCopy2
+		dst[i+1] = uint8(offset)
+		dst[i+2] = uint8(offset >> 8)
+		i += 3
+		length -= x
+	}
+	return i
+}
+
+// Encode returns the encoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire encoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Encode(dst, src []byte) ([]byte, error) {
+	if n := MaxEncodedLen(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	// The block starts with the varint-encoded length of the decompressed bytes.
+	d := binary.PutUvarint(dst, uint64(len(src)))
+
+	// Return early if src is short.
+	if len(src) <= 4 {
+		if len(src) != 0 {
+			d += emitLiteral(dst[d:], src)
+		}
+		return dst[:d], nil
+	}
+
+	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
+	const maxTableSize = 1 << 14
+	shift, tableSize := uint(32-8), 1<<8
+	for tableSize < maxTableSize && tableSize < len(src) {
+		shift--
+		tableSize *= 2
+	}
+	var table [maxTableSize]int
+
+	// Iterate over the source bytes.
+	var (
+		s   int // The iterator position.
+		t   int // The last position with the same hash as s.
+		lit int // The start position of any pending literal bytes.
+	)
+	for s+3 < len(src) {
+		// Update the hash table.
+		b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
+		h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
+		p := &table[(h*0x1e35a7bd)>>shift]
+		// We need to to store values in [-1, inf) in table. To save
+		// some initialization time, (re)use the table's zero value
+		// and shift the values against this zero: add 1 on writes,
+		// subtract 1 on reads.
+		t, *p = *p-1, s+1
+		// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
+		if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
+			s++
+			continue
+		}
+		// Otherwise, we have a match. First, emit any pending literal bytes.
+		if lit != s {
+			d += emitLiteral(dst[d:], src[lit:s])
+		}
+		// Extend the match to be as long as possible.
+		s0 := s
+		s, t = s+4, t+4
+		for s < len(src) && src[s] == src[t] {
+			s++
+			t++
+		}
+		// Emit the copied bytes.
+		d += emitCopy(dst[d:], s-t, s-s0)
+		lit = s
+	}
+
+	// Emit any final pending literal bytes and return.
+	if lit != len(src) {
+		d += emitLiteral(dst[d:], src[lit:])
+	}
+	return dst[:d], nil
+}
+
+// MaxEncodedLen returns the maximum length of a snappy block, given its
+// uncompressed length.
+func MaxEncodedLen(srcLen int) int {
+	// Compressed data can be defined as:
+	//    compressed := item* literal*
+	//    item       := literal* copy
+	//
+	// The trailing literal sequence has a space blowup of at most 62/60
+	// since a literal of length 60 needs one tag byte + one extra byte
+	// for length information.
+	//
+	// Item blowup is trickier to measure. Suppose the "copy" op copies
+	// 4 bytes of data. Because of a special check in the encoding code,
+	// we produce a 4-byte copy only if the offset is < 65536. Therefore
+	// the copy op takes 3 bytes to encode, and this type of item leads
+	// to at most the 62/60 blowup for representing literals.
+	//
+	// Suppose the "copy" op copies 5 bytes of data. If the offset is big
+	// enough, it will take 5 bytes to encode the copy op. Therefore the
+	// worst case here is a one-byte literal followed by a five-byte copy.
+	// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
+	//
+	// This last factor dominates the blowup, so the final estimate is:
+	return 32 + srcLen + srcLen/6
+}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy.go

@@ -0,0 +1,38 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package snappy implements the snappy block-based compression format.
+// It aims for very high speeds and reasonable compression.
+//
+// The C++ snappy implementation is at http://code.google.com/p/snappy/
+package snappy
+
+/*
+Each encoded block begins with the varint-encoded length of the decoded data,
+followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
+first byte of each chunk is broken into its 2 least and 6 most significant bits
+called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
+Zero means a literal tag. All other values mean a copy tag.
+
+For literal tags:
+  - If m < 60, the next 1 + m bytes are literal bytes.
+  - Otherwise, let n be the little-endian unsigned integer denoted by the next
+    m - 59 bytes. The next 1 + n bytes after that are literal bytes.
+
+For copy tags, length bytes are copied from offset bytes ago, in the style of
+Lempel-Ziv compression algorithms. In particular:
+  - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
+    The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
+    of the offset. The next byte is bits 0-7 of the offset.
+  - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
+    The length is 1 + m. The offset is the little-endian unsigned integer
+    denoted by the next 2 bytes.
+  - For l == 3, this tag is a legacy format that is no longer supported.
+*/
+const (
+	tagLiteral = 0x00
+	tagCopy1   = 0x01
+	tagCopy2   = 0x02
+	tagCopy4   = 0x03
+)

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy_test.go

@@ -0,0 +1,261 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+
+func roundtrip(b, ebuf, dbuf []byte) error {
+	e, err := Encode(ebuf, b)
+	if err != nil {
+		return fmt.Errorf("encoding error: %v", err)
+	}
+	d, err := Decode(dbuf, e)
+	if err != nil {
+		return fmt.Errorf("decoding error: %v", err)
+	}
+	if !bytes.Equal(b, d) {
+		return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot  %v", b, d)
+	}
+	return nil
+}
+
+func TestEmpty(t *testing.T) {
+	if err := roundtrip(nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSmallCopy(t *testing.T) {
+	for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
+		for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
+			for i := 0; i < 32; i++ {
+				s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
+				if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
+					t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
+				}
+			}
+		}
+	}
+}
+
+func TestSmallRand(t *testing.T) {
+	rand.Seed(27354294)
+	for n := 1; n < 20000; n += 23 {
+		b := make([]byte, n)
+		for i, _ := range b {
+			b[i] = uint8(rand.Uint32())
+		}
+		if err := roundtrip(b, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestSmallRegular(t *testing.T) {
+	for n := 1; n < 20000; n += 23 {
+		b := make([]byte, n)
+		for i, _ := range b {
+			b[i] = uint8(i%10 + 'a')
+		}
+		if err := roundtrip(b, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func benchDecode(b *testing.B, src []byte) {
+	encoded, err := Encode(nil, src)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// Bandwidth is in amount of uncompressed data.
+	b.SetBytes(int64(len(src)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Decode(src, encoded)
+	}
+}
+
+func benchEncode(b *testing.B, src []byte) {
+	// Bandwidth is in amount of uncompressed data.
+	b.SetBytes(int64(len(src)))
+	dst := make([]byte, MaxEncodedLen(len(src)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Encode(dst, src)
+	}
+}
+
+func readFile(b *testing.B, filename string) []byte {
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		b.Fatalf("failed reading %s: %s", filename, err)
+	}
+	if len(src) == 0 {
+		b.Fatalf("%s has zero length", filename)
+	}
+	return src
+}
+
+// expand returns a slice of length n containing repeated copies of src.
+func expand(src []byte, n int) []byte {
+	dst := make([]byte, n)
+	for x := dst; len(x) > 0; {
+		i := copy(x, src)
+		x = x[i:]
+	}
+	return dst
+}
+
+func benchWords(b *testing.B, n int, decode bool) {
+	// Note: the file is OS-language dependent so the resulting values are not
+	// directly comparable for non-US-English OS installations.
+	data := expand(readFile(b, "/usr/share/dict/words"), n)
+	if decode {
+		benchDecode(b, data)
+	} else {
+		benchEncode(b, data)
+	}
+}
+
+func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
+func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
+func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
+func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
+func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
+func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
+func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
+func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
+
+// testFiles' values are copied directly from
+// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
+// The label field is unused in snappy-go.
+var testFiles = []struct {
+	label    string
+	filename string
+}{
+	{"html", "html"},
+	{"urls", "urls.10K"},
+	{"jpg", "house.jpg"},
+	{"pdf", "mapreduce-osdi-1.pdf"},
+	{"html4", "html_x_4"},
+	{"cp", "cp.html"},
+	{"c", "fields.c"},
+	{"lsp", "grammar.lsp"},
+	{"xls", "kennedy.xls"},
+	{"txt1", "alice29.txt"},
+	{"txt2", "asyoulik.txt"},
+	{"txt3", "lcet10.txt"},
+	{"txt4", "plrabn12.txt"},
+	{"bin", "ptt5"},
+	{"sum", "sum"},
+	{"man", "xargs.1"},
+	{"pb", "geo.protodata"},
+	{"gaviota", "kppkn.gtb"},
+}
+
+// The test data files are present at this canonical URL.
+const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
+
+func downloadTestdata(basename string) (errRet error) {
+	filename := filepath.Join("testdata", basename)
+	f, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("failed to create %s: %s", filename, err)
+	}
+	defer f.Close()
+	defer func() {
+		if errRet != nil {
+			os.Remove(filename)
+		}
+	}()
+	resp, err := http.Get(baseURL + basename)
+	if err != nil {
+		return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to write %s: %s", filename, err)
+	}
+	return nil
+}
+
+func benchFile(b *testing.B, n int, decode bool) {
+	filename := filepath.Join("testdata", testFiles[n].filename)
+	if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
+		if !*download {
+			b.Fatal("test data not found; skipping benchmark without the -download flag")
+		}
+		// Download the official snappy C++ implementation reference test data
+		// files for benchmarking.
+		if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
+			b.Fatalf("failed to create testdata: %s", err)
+		}
+		for _, tf := range testFiles {
+			if err := downloadTestdata(tf.filename); err != nil {
+				b.Fatalf("failed to download testdata: %s", err)
+			}
+		}
+	}
+	data := readFile(b, filename)
+	if decode {
+		benchDecode(b, data)
+	} else {
+		benchEncode(b, data)
+	}
+}
+
+// Naming convention is kept similar to what snappy's C++ implementation uses.
+func Benchmark_UFlat0(b *testing.B)  { benchFile(b, 0, true) }
+func Benchmark_UFlat1(b *testing.B)  { benchFile(b, 1, true) }
+func Benchmark_UFlat2(b *testing.B)  { benchFile(b, 2, true) }
+func Benchmark_UFlat3(b *testing.B)  { benchFile(b, 3, true) }
+func Benchmark_UFlat4(b *testing.B)  { benchFile(b, 4, true) }
+func Benchmark_UFlat5(b *testing.B)  { benchFile(b, 5, true) }
+func Benchmark_UFlat6(b *testing.B)  { benchFile(b, 6, true) }
+func Benchmark_UFlat7(b *testing.B)  { benchFile(b, 7, true) }
+func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
+func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
+func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
+func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
+func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
+func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
+func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
+func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
+func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
+func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
+func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
+func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
+func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
+func Benchmark_ZFlat3(b *testing.B)  { benchFile(b, 3, false) }
+func Benchmark_ZFlat4(b *testing.B)  { benchFile(b, 4, false) }
+func Benchmark_ZFlat5(b *testing.B)  { benchFile(b, 5, false) }
+func Benchmark_ZFlat6(b *testing.B)  { benchFile(b, 6, false) }
+func Benchmark_ZFlat7(b *testing.B)  { benchFile(b, 7, false) }
+func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
+func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
+func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
+func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
+func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
+func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
+func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
+func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
+func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
+func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+)
+
+var (
+	errBatchTooShort  = errors.New("leveldb: batch is too short")
+	errBatchBadRecord = errors.New("leveldb: bad record in batch")
+)
+
+const kBatchHdrLen = 8 + 4
+
+type batchReplay interface {
+	put(key, value []byte, seq uint64)
+	delete(key []byte, seq uint64)
+}
+
+// Batch is a write batch.
+type Batch struct {
+	buf        []byte
+	rLen, bLen int
+	seq        uint64
+	sync       bool
+}
+
+func (b *Batch) grow(n int) {
+	off := len(b.buf)
+	if off == 0 {
+		// include headers
+		off = kBatchHdrLen
+		n += off
+	}
+	if cap(b.buf)-off >= n {
+		return
+	}
+	buf := make([]byte, 2*cap(b.buf)+n)
+	copy(buf, b.buf)
+	b.buf = buf[:off]
+}
+
+func (b *Batch) appendRec(t vType, key, value []byte) {
+	n := 1 + binary.MaxVarintLen32 + len(key)
+	if t == tVal {
+		n += binary.MaxVarintLen32 + len(value)
+	}
+	b.grow(n)
+	off := len(b.buf)
+	buf := b.buf[:off+n]
+	buf[off] = byte(t)
+	off += 1
+	off += binary.PutUvarint(buf[off:], uint64(len(key)))
+	copy(buf[off:], key)
+	off += len(key)
+	if t == tVal {
+		off += binary.PutUvarint(buf[off:], uint64(len(value)))
+		copy(buf[off:], value)
+		off += len(value)
+	}
+	b.buf = buf[:off]
+	b.rLen++
+	//  Include 8-byte ikey header
+	b.bLen += len(key) + len(value) + 8
+}
+
+// Put appends 'put operation' of the given key/value pair to the batch.
+// It is safe to modify the contents of the argument after Put returns.
+func (b *Batch) Put(key, value []byte) {
+	b.appendRec(tVal, key, value)
+}
+
+// Delete appends 'delete operation' of the given key to the batch.
+// It is safe to modify the contents of the argument after Delete returns.
+func (b *Batch) Delete(key []byte) {
+	b.appendRec(tDel, key, nil)
+}
+
+// Reset resets the batch.
+func (b *Batch) Reset() {
+	b.buf = nil
+	b.seq = 0
+	b.rLen = 0
+	b.bLen = 0
+	b.sync = false
+}
+
+func (b *Batch) init(sync bool) {
+	b.sync = sync
+}
+
+func (b *Batch) put(key, value []byte, seq uint64) {
+	if b.rLen == 0 {
+		b.seq = seq
+	}
+	b.Put(key, value)
+}
+
+func (b *Batch) delete(key []byte, seq uint64) {
+	if b.rLen == 0 {
+		b.seq = seq
+	}
+	b.Delete(key)
+}
+
+func (b *Batch) append(p *Batch) {
+	if p.rLen > 0 {
+		b.grow(len(p.buf) - kBatchHdrLen)
+		b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
+		b.rLen += p.rLen
+	}
+	if p.sync {
+		b.sync = true
+	}
+}
+
+func (b *Batch) len() int {
+	return b.rLen
+}
+
+func (b *Batch) size() int {
+	return b.bLen
+}
+
+func (b *Batch) encode() []byte {
+	b.grow(0)
+	binary.LittleEndian.PutUint64(b.buf, b.seq)
+	binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
+
+	return b.buf
+}
+
+func (b *Batch) decode(buf []byte) error {
+	if len(buf) < kBatchHdrLen {
+		return errBatchTooShort
+	}
+
+	b.seq = binary.LittleEndian.Uint64(buf)
+	b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
+	// No need to be precise at this point, it won't be used anyway
+	b.bLen = len(buf) - kBatchHdrLen
+	b.buf = buf
+
+	return nil
+}
+
+func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
+	off := kBatchHdrLen
+	for i := 0; i < b.rLen; i++ {
+		if off >= len(b.buf) {
+			return errors.New("leveldb: invalid batch record length")
+		}
+
+		t := vType(b.buf[off])
+		if t > tVal {
+			return errors.New("leveldb: invalid batch record type in batch")
+		}
+		off += 1
+
+		x, n := binary.Uvarint(b.buf[off:])
+		off += n
+		if n <= 0 || off+int(x) > len(b.buf) {
+			return errBatchBadRecord
+		}
+		key := b.buf[off : off+int(x)]
+		off += int(x)
+
+		var value []byte
+		if t == tVal {
+			x, n := binary.Uvarint(b.buf[off:])
+			off += n
+			if n <= 0 || off+int(x) > len(b.buf) {
+				return errBatchBadRecord
+			}
+			value = b.buf[off : off+int(x)]
+			off += int(x)
+		}
+
+		f(i, t, key, value)
+	}
+
+	return nil
+}
+
+func (b *Batch) replay(to batchReplay) error {
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		switch t {
+		case tVal:
+			to.put(key, value, b.seq+uint64(i))
+		case tDel:
+			to.delete(key, b.seq+uint64(i))
+		}
+	})
+}
+
+func (b *Batch) memReplay(to *memdb.DB) error {
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		ikey := newIKey(key, b.seq+uint64(i), t)
+		to.Put(ikey, value)
+	})
+}
+
+func (b *Batch) revertMemReplay(to *memdb.DB) error {
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		ikey := newIKey(key, b.seq+uint64(i), t)
+		to.Delete(ikey)
+	})
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+)
+
+type tbRec struct {
+	t          vType
+	key, value []byte
+}
+
+type testBatch struct {
+	rec []*tbRec
+}
+
+func (p *testBatch) put(key, value []byte, seq uint64) {
+	p.rec = append(p.rec, &tbRec{tVal, key, value})
+}
+
+func (p *testBatch) delete(key []byte, seq uint64) {
+	p.rec = append(p.rec, &tbRec{tDel, key, nil})
+}
+
+func compareBatch(t *testing.T, b1, b2 *Batch) {
+	if b1.seq != b2.seq {
+		t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
+	}
+	if b1.len() != b2.len() {
+		t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
+	}
+	p1, p2 := new(testBatch), new(testBatch)
+	err := b1.replay(p1)
+	if err != nil {
+		t.Fatal("error when replaying batch 1: ", err)
+	}
+	err = b2.replay(p2)
+	if err != nil {
+		t.Fatal("error when replaying batch 2: ", err)
+	}
+	for i := range p1.rec {
+		r1, r2 := p1.rec[i], p2.rec[i]
+		if r1.t != r2.t {
+			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
+		}
+		if !bytes.Equal(r1.key, r2.key) {
+			t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
+		}
+		if r1.t == tVal {
+			if !bytes.Equal(r1.value, r2.value) {
+				t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
+			}
+		}
+	}
+}
+
+func TestBatch_EncodeDecode(t *testing.T) {
+	b1 := new(Batch)
+	b1.seq = 10009
+	b1.Put([]byte("key1"), []byte("value1"))
+	b1.Put([]byte("key2"), []byte("value2"))
+	b1.Delete([]byte("key1"))
+	b1.Put([]byte("k"), []byte(""))
+	b1.Put([]byte("zzzzzzzzzzz"), []byte("zzzzzzzzzzzzzzzzzzzzzzzz"))
+	b1.Delete([]byte("key10000"))
+	b1.Delete([]byte("k"))
+	buf := b1.encode()
+	b2 := new(Batch)
+	err := b2.decode(buf)
+	if err != nil {
+		t.Error("error when decoding batch: ", err)
+	}
+	compareBatch(t, b1, b2)
+}
+
+func TestBatch_Append(t *testing.T) {
+	b1 := new(Batch)
+	b1.seq = 10009
+	b1.Put([]byte("key1"), []byte("value1"))
+	b1.Put([]byte("key2"), []byte("value2"))
+	b1.Delete([]byte("key1"))
+	b1.Put([]byte("foo"), []byte("foovalue"))
+	b1.Put([]byte("bar"), []byte("barvalue"))
+	b2a := new(Batch)
+	b2a.seq = 10009
+	b2a.Put([]byte("key1"), []byte("value1"))
+	b2a.Put([]byte("key2"), []byte("value2"))
+	b2a.Delete([]byte("key1"))
+	b2b := new(Batch)
+	b2b.Put([]byte("foo"), []byte("foovalue"))
+	b2b.Put([]byte("bar"), []byte("barvalue"))
+	b2a.append(b2b)
+	compareBatch(t, b1, b2a)
+}
+
+func TestBatch_Size(t *testing.T) {
+	b := new(Batch)
+	for i := 0; i < 2; i++ {
+		b.Put([]byte("key1"), []byte("value1"))
+		b.Put([]byte("key2"), []byte("value2"))
+		b.Delete([]byte("key1"))
+		b.Put([]byte("foo"), []byte("foovalue"))
+		b.Put([]byte("bar"), []byte("barvalue"))
+		mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
+		b.memReplay(mem)
+		if b.size() != mem.Size() {
+			t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
+		}
+		b.Reset()
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go

@@ -0,0 +1,461 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func randomString(r *rand.Rand, n int) []byte {
+	b := new(bytes.Buffer)
+	for i := 0; i < n; i++ {
+		b.WriteByte(' ' + byte(r.Intn(95)))
+	}
+	return b.Bytes()
+}
+
+func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
+	nn := int(float32(n) * frac)
+	rb := randomString(r, nn)
+	b := make([]byte, 0, n+nn)
+	for len(b) < n {
+		b = append(b, rb...)
+	}
+	return b[:n]
+}
+
+type valueGen struct {
+	src []byte
+	pos int
+}
+
+func newValueGen(frac float32) *valueGen {
+	v := new(valueGen)
+	r := rand.New(rand.NewSource(301))
+	v.src = make([]byte, 0, 1048576+100)
+	for len(v.src) < 1048576 {
+		v.src = append(v.src, compressibleStr(r, frac, 100)...)
+	}
+	return v
+}
+
+func (v *valueGen) get(n int) []byte {
+	if v.pos+n > len(v.src) {
+		v.pos = 0
+	}
+	v.pos += n
+	return v.src[v.pos-n : v.pos]
+}
+
+var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
+
+type dbBench struct {
+	b    *testing.B
+	stor storage.Storage
+	db   *DB
+
+	o  *opt.Options
+	ro *opt.ReadOptions
+	wo *opt.WriteOptions
+
+	keys, values [][]byte
+}
+
+func openDBBench(b *testing.B, noCompress bool) *dbBench {
+	_, err := os.Stat(benchDB)
+	if err == nil {
+		err = os.RemoveAll(benchDB)
+		if err != nil {
+			b.Fatal("cannot remove old db: ", err)
+		}
+	}
+
+	p := &dbBench{
+		b:  b,
+		o:  &opt.Options{},
+		ro: &opt.ReadOptions{},
+		wo: &opt.WriteOptions{},
+	}
+	p.stor, err = storage.OpenFile(benchDB)
+	if err != nil {
+		b.Fatal("cannot open stor: ", err)
+	}
+	if noCompress {
+		p.o.Compression = opt.NoCompression
+	}
+
+	p.db, err = Open(p.stor, p.o)
+	if err != nil {
+		b.Fatal("cannot open db: ", err)
+	}
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	return p
+}
+
+func (p *dbBench) reopen() {
+	p.db.Close()
+	var err error
+	p.db, err = Open(p.stor, p.o)
+	if err != nil {
+		p.b.Fatal("Reopen: got error: ", err)
+	}
+}
+
+func (p *dbBench) populate(n int) {
+	p.keys, p.values = make([][]byte, n), make([][]byte, n)
+	v := newValueGen(0.5)
+	for i := range p.keys {
+		p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
+	}
+}
+
+func (p *dbBench) randomize() {
+	m := len(p.keys)
+	times := m * 2
+	r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
+	for n := 0; n < times; n++ {
+		i, j := r1.Int()%m, r2.Int()%m
+		if i == j {
+			continue
+		}
+		p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
+		p.values[i], p.values[j] = p.values[j], p.values[i]
+	}
+}
+
+func (p *dbBench) writes(perBatch int) {
+	b := p.b
+	db := p.db
+
+	n := len(p.keys)
+	m := n / perBatch
+	if n%perBatch > 0 {
+		m++
+	}
+	batches := make([]Batch, m)
+	j := 0
+	for i := range batches {
+		first := true
+		for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
+			first = false
+			batches[i].Put(p.keys[j], p.values[j])
+		}
+	}
+	runtime.GC()
+
+	b.ResetTimer()
+	b.StartTimer()
+	for i := range batches {
+		err := db.Write(&(batches[i]), p.wo)
+		if err != nil {
+			b.Fatal("write failed: ", err)
+		}
+	}
+	b.StopTimer()
+	b.SetBytes(116)
+}
+
+func (p *dbBench) drop() {
+	p.keys, p.values = nil, nil
+	runtime.GC()
+}
+
+func (p *dbBench) puts() {
+	b := p.b
+	db := p.db
+
+	b.ResetTimer()
+	b.StartTimer()
+	for i := range p.keys {
+		err := db.Put(p.keys[i], p.values[i], p.wo)
+		if err != nil {
+			b.Fatal("put failed: ", err)
+		}
+	}
+	b.StopTimer()
+	b.SetBytes(116)
+}
+
+func (p *dbBench) fill() {
+	b := p.b
+	db := p.db
+
+	perBatch := 10000
+	batch := new(Batch)
+	for i, n := 0, len(p.keys); i < n; {
+		first := true
+		for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
+			first = false
+			batch.Put(p.keys[i], p.values[i])
+		}
+		err := db.Write(batch, p.wo)
+		if err != nil {
+			b.Fatal("write failed: ", err)
+		}
+		batch.Reset()
+	}
+}
+
+func (p *dbBench) gets() {
+	b := p.b
+	db := p.db
+
+	b.ResetTimer()
+	for i := range p.keys {
+		_, err := db.Get(p.keys[i], p.ro)
+		if err != nil {
+			b.Error("got error: ", err)
+		}
+	}
+	b.StopTimer()
+}
+
+func (p *dbBench) seeks() {
+	b := p.b
+
+	iter := p.newIter()
+	defer iter.Release()
+	b.ResetTimer()
+	for i := range p.keys {
+		if !iter.Seek(p.keys[i]) {
+			b.Error("value not found for: ", string(p.keys[i]))
+		}
+	}
+	b.StopTimer()
+}
+
+func (p *dbBench) newIter() iterator.Iterator {
+	iter := p.db.NewIterator(nil, p.ro)
+	err := iter.Error()
+	if err != nil {
+		p.b.Fatal("cannot create iterator: ", err)
+	}
+	return iter
+}
+
+func (p *dbBench) close() {
+	p.db.Close()
+	p.stor.Close()
+	os.RemoveAll(benchDB)
+	p.db = nil
+	p.keys = nil
+	p.values = nil
+	runtime.GC()
+	runtime.GOMAXPROCS(1)
+}
+
+func BenchmarkDBWrite(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBWriteBatch(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.writes(1000)
+	p.close()
+}
+
+func BenchmarkDBWriteUncompressed(b *testing.B) {
+	p := openDBBench(b, true)
+	p.populate(b.N)
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
+	p := openDBBench(b, true)
+	p.populate(b.N)
+	p.writes(1000)
+	p.close()
+}
+
+func BenchmarkDBWriteRandom(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.randomize()
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBWriteRandomSync(b *testing.B) {
+	p := openDBBench(b, false)
+	p.wo.Sync = true
+	p.populate(b.N)
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBOverwrite(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.writes(1)
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBOverwriteRandom(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.writes(1)
+	p.randomize()
+	p.writes(1)
+	p.close()
+}
+
+func BenchmarkDBPut(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.puts()
+	p.close()
+}
+
+func BenchmarkDBRead(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.drop()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	for iter.Next() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBReadGC(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	for iter.Next() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBReadUncompressed(b *testing.B) {
+	p := openDBBench(b, true)
+	p.populate(b.N)
+	p.fill()
+	p.drop()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	for iter.Next() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBReadTable(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.reopen()
+	p.drop()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	for iter.Next() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBReadReverse(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.drop()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	iter.Last()
+	for iter.Prev() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBReadReverseTable(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.reopen()
+	p.drop()
+
+	iter := p.newIter()
+	b.ResetTimer()
+	iter.Last()
+	for iter.Prev() {
+	}
+	iter.Release()
+	b.StopTimer()
+	b.SetBytes(116)
+	p.close()
+}
+
+func BenchmarkDBSeek(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.seeks()
+	p.close()
+}
+
+func BenchmarkDBSeekRandom(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.randomize()
+	p.seeks()
+	p.close()
+}
+
+func BenchmarkDBGet(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gets()
+	p.close()
+}
+
+func BenchmarkDBGetRandom(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.randomize()
+	p.gets()
+	p.close()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package cache provides interface and implementation of a cache algorithms.
+package cache
+
+import (
+	"sync/atomic"
+)
+
+// SetFunc used by Namespace.Get method to create a cache object. SetFunc
+// may return ok false, in that case the cache object will not be created.
+type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
+
+// SetFin will be called when corresponding cache object are released.
+type SetFin func()
+
+// DelFin will be called when corresponding cache object are released.
+// DelFin will be called after SetFin. The exist is true if the corresponding
+// cache object is actually exist in the cache tree.
+type DelFin func(exist bool)
+
+// PurgeFin will be called when corresponding cache object are released.
+// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
+// not be executed but passed to the PurgeFin, it is up to the caller
+// to call it or not.
+type PurgeFin func(ns, key uint64, delfin DelFin)
+
+// Cache is a cache tree.
+type Cache interface {
+	// SetCapacity sets cache capacity.
+	SetCapacity(capacity int)
+
+	// GetNamespace gets or creates a cache namespace for the given id.
+	GetNamespace(id uint64) Namespace
+
+	// Purge purges all cache namespaces, read Namespace.Purge method documentation.
+	Purge(fin PurgeFin)
+
+	// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
+	Zap(closed bool)
+}
+
+// Namespace is a cache namespace.
+type Namespace interface {
+	// Get gets cache object for the given key. The given SetFunc (if not nil) will
+	// be called if the given key does not exist.
+	// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
+	// will return ok false.
+	Get(key uint64, setf SetFunc) (obj Object, ok bool)
+
+	// Get deletes cache object for the given key. If exist the cache object will
+	// be deleted later when all of its handles have been released (i.e. no one use
+	// it anymore) and the given DelFin (if not nil) will finally be  executed. If
+	// such cache object does not exist the given DelFin will be executed anyway.
+	//
+	// Delete returns true if such cache object exist.
+	Delete(key uint64, fin DelFin) bool
+
+	// Purge deletes all cache objects, read Delete method documentation.
+	Purge(fin PurgeFin)
+
+	// Zap detaches the namespace from the cache tree and delete all its cache
+	// objects. The cache objects deletion and finalizers execution are happen
+	// immediately, even if its existing handles haven't yet been released.
+	// A zapped namespace can't never be filled again.
+	// If closed is false then the Get function will always call the given SetFunc
+	// if it is not nil, but resultant of the SetFunc will not be cached.
+	Zap(closed bool)
+}
+
+// Object is a cache object.
+type Object interface {
+	// Release releases the cache object. Other methods should not be called
+	// after the cache object has been released.
+	Release()
+
+	// Value returns value of the cache object.
+	Value() interface{}
+}
+
+// Namespace state.
+type nsState int
+
+const (
+	nsEffective nsState = iota
+	nsZapped
+	nsClosed
+)
+
+// Node state.
+type nodeState int
+
+const (
+	nodeEffective nodeState = iota
+	nodeEvicted
+	nodeRemoved
+)
+
+// Fake object.
+type fakeObject struct {
+	value interface{}
+	fin   func()
+	once  uint32
+}
+
+func (o *fakeObject) Value() interface{} {
+	if atomic.LoadUint32(&o.once) == 0 {
+		return o.value
+	}
+	return nil
+}
+
+func (o *fakeObject) Release() {
+	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+		return
+	}
+	if o.fin != nil {
+		o.fin()
+		o.fin = nil
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -0,0 +1,236 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
+	obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
+		return true, value, charge, fin
+	})
+	return obj
+}
+
+func TestCache_HitMiss(t *testing.T) {
+	cases := []struct {
+		key   uint64
+		value string
+	}{
+		{1, "vvvvvvvvv"},
+		{100, "v1"},
+		{0, "v2"},
+		{12346, "v3"},
+		{777, "v4"},
+		{999, "v5"},
+		{7654, "v6"},
+		{2, "v7"},
+		{3, "v8"},
+		{9, "v9"},
+	}
+
+	setfin := 0
+	c := NewLRUCache(1000)
+	ns := c.GetNamespace(0)
+	for i, x := range cases {
+		set(ns, x.key, x.value, len(x.value), func() {
+			setfin++
+		}).Release()
+		for j, y := range cases {
+			r, ok := ns.Get(y.key, nil)
+			if j <= i {
+				// should hit
+				if !ok {
+					t.Errorf("case '%d' iteration '%d' is miss", i, j)
+				} else if r.Value().(string) != y.value {
+					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				}
+			} else {
+				// should miss
+				if ok {
+					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
+				}
+			}
+			if ok {
+				r.Release()
+			}
+		}
+	}
+
+	for i, x := range cases {
+		finalizerOk := false
+		ns.Delete(x.key, func(exist bool) {
+			finalizerOk = true
+		})
+
+		if !finalizerOk {
+			t.Errorf("case %d delete finalizer not executed", i)
+		}
+
+		for j, y := range cases {
+			r, ok := ns.Get(y.key, nil)
+			if j > i {
+				// should hit
+				if !ok {
+					t.Errorf("case '%d' iteration '%d' is miss", i, j)
+				} else if r.Value().(string) != y.value {
+					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				}
+			} else {
+				// should miss
+				if ok {
+					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
+				}
+			}
+			if ok {
+				r.Release()
+			}
+		}
+	}
+
+	if setfin != len(cases) {
+		t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin)
+	}
+}
+
+func TestLRUCache_Eviction(t *testing.T) {
+	c := NewLRUCache(12)
+	ns := c.GetNamespace(0)
+	o1 := set(ns, 1, 1, 1, nil)
+	set(ns, 2, 2, 1, nil).Release()
+	set(ns, 3, 3, 1, nil).Release()
+	set(ns, 4, 4, 1, nil).Release()
+	set(ns, 5, 5, 1, nil).Release()
+	if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
+		r.Release()
+	}
+	set(ns, 9, 9, 10, nil).Release() // 5,2,9
+
+	for _, x := range []uint64{9, 2, 5, 1} {
+		r, ok := ns.Get(x, nil)
+		if !ok {
+			t.Errorf("miss for key '%d'", x)
+		} else {
+			if r.Value().(int) != int(x) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			}
+			r.Release()
+		}
+	}
+	o1.Release()
+	for _, x := range []uint64{1, 2, 5} {
+		r, ok := ns.Get(x, nil)
+		if !ok {
+			t.Errorf("miss for key '%d'", x)
+		} else {
+			if r.Value().(int) != int(x) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			}
+			r.Release()
+		}
+	}
+	for _, x := range []uint64{3, 4, 9} {
+		r, ok := ns.Get(x, nil)
+		if ok {
+			t.Errorf("hit for key '%d'", x)
+			if r.Value().(int) != int(x) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			}
+			r.Release()
+		}
+	}
+}
+
+func TestLRUCache_SetGet(t *testing.T) {
+	c := NewLRUCache(13)
+	ns := c.GetNamespace(0)
+	for i := 0; i < 200; i++ {
+		n := uint64(rand.Intn(99999) % 20)
+		set(ns, n, n, 1, nil).Release()
+		if p, ok := ns.Get(n, nil); ok {
+			if p.Value() == nil {
+				t.Errorf("key '%d' contains nil value", n)
+			} else {
+				got := p.Value().(uint64)
+				if got != n {
+					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
+				}
+			}
+			p.Release()
+		} else {
+			t.Errorf("key '%d' doesn't exist", n)
+		}
+	}
+}
+
+func TestLRUCache_Purge(t *testing.T) {
+	c := NewLRUCache(3)
+	ns1 := c.GetNamespace(0)
+	o1 := set(ns1, 1, 1, 1, nil)
+	o2 := set(ns1, 2, 2, 1, nil)
+	ns1.Purge(nil)
+	set(ns1, 3, 3, 1, nil).Release()
+	for _, x := range []uint64{1, 2, 3} {
+		r, ok := ns1.Get(x, nil)
+		if !ok {
+			t.Errorf("miss for key '%d'", x)
+		} else {
+			if r.Value().(int) != int(x) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			}
+			r.Release()
+		}
+	}
+	o1.Release()
+	o2.Release()
+	for _, x := range []uint64{1, 2} {
+		r, ok := ns1.Get(x, nil)
+		if ok {
+			t.Errorf("hit for key '%d'", x)
+			if r.Value().(int) != int(x) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			}
+			r.Release()
+		}
+	}
+}
+
+func BenchmarkLRUCache_SetRelease(b *testing.B) {
+	capacity := b.N / 100
+	if capacity <= 0 {
+		capacity = 10
+	}
+	c := NewLRUCache(capacity)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, nil, 1, nil).Release()
+	}
+}
+
+func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
+	capacity := b.N / 100
+	if capacity <= 0 {
+		capacity = 10
+	}
+	c := NewLRUCache(capacity)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+
+	na := b.N / 2
+	nb := b.N - na
+
+	for i := uint64(0); i < uint64(na); i++ {
+		set(ns, i, nil, 1, nil).Release()
+	}
+
+	for i := uint64(0); i < uint64(nb); i++ {
+		set(ns, i, nil, 1, nil).Release()
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+type emptyCache struct {
+	sync.Mutex
+	table map[uint64]*emptyNS
+}
+
+// NewEmptyCache creates a new initialized empty cache.
+func NewEmptyCache() Cache {
+	return &emptyCache{
+		table: make(map[uint64]*emptyNS),
+	}
+}
+
+func (c *emptyCache) GetNamespace(id uint64) Namespace {
+	c.Lock()
+	defer c.Unlock()
+
+	if ns, ok := c.table[id]; ok {
+		return ns
+	}
+
+	ns := &emptyNS{
+		cache: c,
+		id:    id,
+		table: make(map[uint64]*emptyNode),
+	}
+	c.table[id] = ns
+	return ns
+}
+
+func (c *emptyCache) Purge(fin PurgeFin) {
+	c.Lock()
+	for _, ns := range c.table {
+		ns.purgeNB(fin)
+	}
+	c.Unlock()
+}
+
+func (c *emptyCache) Zap(closed bool) {
+	c.Lock()
+	for _, ns := range c.table {
+		ns.zapNB(closed)
+	}
+	c.table = make(map[uint64]*emptyNS)
+	c.Unlock()
+}
+
+func (*emptyCache) SetCapacity(capacity int) {}
+
+type emptyNS struct {
+	cache *emptyCache
+	id    uint64
+	table map[uint64]*emptyNode
+	state nsState
+}
+
+func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
+	ns.cache.Lock()
+
+	switch ns.state {
+	case nsZapped:
+		ns.cache.Unlock()
+		if setf == nil {
+			return
+		}
+
+		var value interface{}
+		var fin func()
+		ok, value, _, fin = setf()
+		if ok {
+			o = &fakeObject{
+				value: value,
+				fin:   fin,
+			}
+		}
+		return
+	case nsClosed:
+		ns.cache.Unlock()
+		return
+	}
+
+	n, ok := ns.table[key]
+	if ok {
+		n.ref++
+	} else {
+		if setf == nil {
+			ns.cache.Unlock()
+			return
+		}
+
+		var value interface{}
+		var fin func()
+		ok, value, _, fin = setf()
+		if !ok {
+			ns.cache.Unlock()
+			return
+		}
+
+		n = &emptyNode{
+			ns:     ns,
+			key:    key,
+			value:  value,
+			setfin: fin,
+			ref:    1,
+		}
+		ns.table[key] = n
+	}
+
+	ns.cache.Unlock()
+	o = &emptyObject{node: n}
+	return
+}
+
+func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
+	ns.cache.Lock()
+
+	if ns.state != nsEffective {
+		ns.cache.Unlock()
+		if fin != nil {
+			fin(false)
+		}
+		return false
+	}
+
+	n, ok := ns.table[key]
+	if !ok {
+		ns.cache.Unlock()
+		if fin != nil {
+			fin(false)
+		}
+		return false
+	}
+	n.delfin = fin
+	ns.cache.Unlock()
+	return true
+}
+
+func (ns *emptyNS) purgeNB(fin PurgeFin) {
+	if ns.state != nsEffective {
+		return
+	}
+	for _, n := range ns.table {
+		n.purgefin = fin
+	}
+}
+
+func (ns *emptyNS) Purge(fin PurgeFin) {
+	ns.cache.Lock()
+	ns.purgeNB(fin)
+	ns.cache.Unlock()
+}
+
+func (ns *emptyNS) zapNB(closed bool) {
+	if ns.state != nsEffective {
+		return
+	}
+	for _, n := range ns.table {
+		n.execFin()
+	}
+	if closed {
+		ns.state = nsClosed
+	} else {
+		ns.state = nsZapped
+	}
+	ns.table = nil
+}
+
+func (ns *emptyNS) Zap(closed bool) {
+	ns.cache.Lock()
+	ns.zapNB(closed)
+	delete(ns.cache.table, ns.id)
+	ns.cache.Unlock()
+}
+
+type emptyNode struct {
+	ns       *emptyNS
+	key      uint64
+	value    interface{}
+	ref      int
+	setfin   SetFin
+	delfin   DelFin
+	purgefin PurgeFin
+}
+
+func (n *emptyNode) execFin() {
+	if n.setfin != nil {
+		n.setfin()
+		n.setfin = nil
+	}
+	if n.purgefin != nil {
+		n.purgefin(n.ns.id, n.key, n.delfin)
+		n.delfin = nil
+		n.purgefin = nil
+	} else if n.delfin != nil {
+		n.delfin(true)
+		n.delfin = nil
+	}
+}
+
+func (n *emptyNode) evict() {
+	n.ns.cache.Lock()
+	n.ref--
+	if n.ref == 0 {
+		if n.ns.state == nsEffective {
+			// Remove elem.
+			delete(n.ns.table, n.key)
+			// Execute finalizer.
+			n.execFin()
+		}
+	} else if n.ref < 0 {
+		panic("leveldb/cache: emptyNode: negative node reference")
+	}
+	n.ns.cache.Unlock()
+}
+
+type emptyObject struct {
+	node *emptyNode
+	once uint32
+}
+
+func (o *emptyObject) Value() interface{} {
+	if atomic.LoadUint32(&o.once) == 0 {
+		return o.node.value
+	}
+	return nil
+}
+
+func (o *emptyObject) Release() {
+	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+		return
+	}
+	o.node.evict()
+	o.node = nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -0,0 +1,354 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// lruCache represent a LRU cache state.
+type lruCache struct {
+	sync.Mutex
+
+	recent   lruNode
+	table    map[uint64]*lruNs
+	capacity int
+	size     int
+}
+
+// NewLRUCache creates a new initialized LRU cache with the given capacity.
+func NewLRUCache(capacity int) Cache {
+	c := &lruCache{
+		table:    make(map[uint64]*lruNs),
+		capacity: capacity,
+	}
+	c.recent.rNext = &c.recent
+	c.recent.rPrev = &c.recent
+	return c
+}
+
+// SetCapacity set cache capacity.
+func (c *lruCache) SetCapacity(capacity int) {
+	c.Lock()
+	c.capacity = capacity
+	c.evict()
+	c.Unlock()
+}
+
+// GetNamespace return namespace object for given id.
+func (c *lruCache) GetNamespace(id uint64) Namespace {
+	c.Lock()
+	defer c.Unlock()
+
+	if p, ok := c.table[id]; ok {
+		return p
+	}
+
+	p := &lruNs{
+		lru:   c,
+		id:    id,
+		table: make(map[uint64]*lruNode),
+	}
+	c.table[id] = p
+	return p
+}
+
+// Purge purge entire cache.
+func (c *lruCache) Purge(fin PurgeFin) {
+	c.Lock()
+	for _, ns := range c.table {
+		ns.purgeNB(fin)
+	}
+	c.Unlock()
+}
+
+func (c *lruCache) Zap(closed bool) {
+	c.Lock()
+	for _, ns := range c.table {
+		ns.zapNB(closed)
+	}
+	c.table = make(map[uint64]*lruNs)
+	c.Unlock()
+}
+
+func (c *lruCache) evict() {
+	top := &c.recent
+	for n := c.recent.rPrev; c.size > c.capacity && n != top; {
+		n.state = nodeEvicted
+		n.rRemove()
+		n.evictNB()
+		c.size -= n.charge
+		n = c.recent.rPrev
+	}
+}
+
+type lruNs struct {
+	lru   *lruCache
+	id    uint64
+	table map[uint64]*lruNode
+	state nsState
+}
+
+func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
+	lru := ns.lru
+	lru.Lock()
+
+	switch ns.state {
+	case nsZapped:
+		lru.Unlock()
+		if setf == nil {
+			return
+		}
+
+		var value interface{}
+		var fin func()
+		ok, value, _, fin = setf()
+		if ok {
+			o = &fakeObject{
+				value: value,
+				fin:   fin,
+			}
+		}
+		return
+	case nsClosed:
+		lru.Unlock()
+		return
+	}
+
+	n, ok := ns.table[key]
+	if ok {
+		switch n.state {
+		case nodeEvicted:
+			// Insert to recent list.
+			n.state = nodeEffective
+			n.ref++
+			lru.size += n.charge
+			lru.evict()
+			fallthrough
+		case nodeEffective:
+			// Bump to front
+			n.rRemove()
+			n.rInsert(&lru.recent)
+		}
+		n.ref++
+	} else {
+		if setf == nil {
+			lru.Unlock()
+			return
+		}
+
+		var value interface{}
+		var charge int
+		var fin func()
+		ok, value, charge, fin = setf()
+		if !ok {
+			lru.Unlock()
+			return
+		}
+
+		n = &lruNode{
+			ns:     ns,
+			key:    key,
+			value:  value,
+			charge: charge,
+			setfin: fin,
+			ref:    2,
+		}
+		ns.table[key] = n
+		n.rInsert(&lru.recent)
+
+		lru.size += charge
+		lru.evict()
+	}
+
+	lru.Unlock()
+	o = &lruObject{node: n}
+	return
+}
+
+func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
+	lru := ns.lru
+	lru.Lock()
+
+	if ns.state != nsEffective {
+		lru.Unlock()
+		if fin != nil {
+			fin(false)
+		}
+		return false
+	}
+
+	n, ok := ns.table[key]
+	if !ok {
+		lru.Unlock()
+		if fin != nil {
+			fin(false)
+		}
+		return false
+	}
+
+	n.delfin = fin
+	switch n.state {
+	case nodeRemoved:
+		lru.Unlock()
+		return false
+	case nodeEffective:
+		lru.size -= n.charge
+		n.rRemove()
+		n.evictNB()
+	}
+	n.state = nodeRemoved
+
+	lru.Unlock()
+	return true
+}
+
+func (ns *lruNs) purgeNB(fin PurgeFin) {
+	lru := ns.lru
+	if ns.state != nsEffective {
+		return
+	}
+
+	for _, n := range ns.table {
+		n.purgefin = fin
+		if n.state == nodeEffective {
+			lru.size -= n.charge
+			n.rRemove()
+			n.evictNB()
+		}
+		n.state = nodeRemoved
+	}
+}
+
+func (ns *lruNs) Purge(fin PurgeFin) {
+	ns.lru.Lock()
+	ns.purgeNB(fin)
+	ns.lru.Unlock()
+}
+
+func (ns *lruNs) zapNB(closed bool) {
+	lru := ns.lru
+	if ns.state != nsEffective {
+		return
+	}
+
+	if closed {
+		ns.state = nsClosed
+	} else {
+		ns.state = nsZapped
+	}
+	for _, n := range ns.table {
+		if n.state == nodeEffective {
+			lru.size -= n.charge
+			n.rRemove()
+		}
+		n.state = nodeRemoved
+		n.execFin()
+	}
+	ns.table = nil
+}
+
+func (ns *lruNs) Zap(closed bool) {
+	ns.lru.Lock()
+	ns.zapNB(closed)
+	delete(ns.lru.table, ns.id)
+	ns.lru.Unlock()
+}
+
+type lruNode struct {
+	ns *lruNs
+
+	rNext, rPrev *lruNode
+
+	key      uint64
+	value    interface{}
+	charge   int
+	ref      int
+	state    nodeState
+	setfin   SetFin
+	delfin   DelFin
+	purgefin PurgeFin
+}
+
+func (n *lruNode) rInsert(at *lruNode) {
+	x := at.rNext
+	at.rNext = n
+	n.rPrev = at
+	n.rNext = x
+	x.rPrev = n
+}
+
+func (n *lruNode) rRemove() bool {
+	// only remove if not already removed
+	if n.rPrev == nil {
+		return false
+	}
+
+	n.rPrev.rNext = n.rNext
+	n.rNext.rPrev = n.rPrev
+	n.rPrev = nil
+	n.rNext = nil
+
+	return true
+}
+
+func (n *lruNode) execFin() {
+	if n.setfin != nil {
+		n.setfin()
+		n.setfin = nil
+	}
+	if n.purgefin != nil {
+		n.purgefin(n.ns.id, n.key, n.delfin)
+		n.delfin = nil
+		n.purgefin = nil
+	} else if n.delfin != nil {
+		n.delfin(true)
+		n.delfin = nil
+	}
+}
+
+func (n *lruNode) evictNB() {
+	n.ref--
+	if n.ref == 0 {
+		if n.ns.state == nsEffective {
+			// remove elem
+			delete(n.ns.table, n.key)
+			// execute finalizer
+			n.execFin()
+		}
+	} else if n.ref < 0 {
+		panic("leveldb/cache: lruCache: negative node reference")
+	}
+}
+
+func (n *lruNode) evict() {
+	n.ns.lru.Lock()
+	n.evictNB()
+	n.ns.lru.Unlock()
+}
+
+type lruObject struct {
+	node *lruNode
+	once uint32
+}
+
+func (o *lruObject) Value() interface{} {
+	if atomic.LoadUint32(&o.once) == 0 {
+		return o.node.value
+	}
+	return nil
+}
+
+func (o *lruObject) Release() {
+	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+		return
+	}
+
+	o.node.evict()
+	o.node = nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import "github.com/syndtr/goleveldb/leveldb/comparer"
+
+type iComparer struct {
+	cmp comparer.Comparer
+}
+
+func (p *iComparer) Name() string {
+	return p.cmp.Name()
+}
+
+func (p *iComparer) Compare(a, b []byte) int {
+	ia, ib := iKey(a), iKey(b)
+	r := p.cmp.Compare(ia.ukey(), ib.ukey())
+	if r == 0 {
+		an, bn := ia.num(), ib.num()
+		if an > bn {
+			r = -1
+		} else if an < bn {
+			r = 1
+		}
+	}
+	return r
+}
+
+func (p *iComparer) Separator(dst, a, b []byte) []byte {
+	ua, ub := iKey(a).ukey(), iKey(b).ukey()
+	dst = p.cmp.Separator(dst, ua, ub)
+	if dst == nil {
+		return nil
+	}
+	if len(dst) < len(ua) && p.cmp.Compare(ua, dst) < 0 {
+		dst = append(dst, kMaxNumBytes...)
+	} else {
+		// Did not close possibilities that n maybe longer than len(ub).
+		dst = append(dst, a[len(a)-8:]...)
+	}
+	return dst
+}
+
+func (p *iComparer) Successor(dst, b []byte) []byte {
+	ub := iKey(b).ukey()
+	dst = p.cmp.Successor(dst, ub)
+	if dst == nil {
+		return nil
+	}
+	if len(dst) < len(ub) && p.cmp.Compare(ub, dst) < 0 {
+		dst = append(dst, kMaxNumBytes...)
+	} else {
+		// Did not close possibilities that n maybe longer than len(ub).
+		dst = append(dst, b[len(b)-8:]...)
+	}
+	return dst
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package comparer
+
+import "bytes"
+
+type bytesComparer struct{}
+
+func (bytesComparer) Compare(a, b []byte) int {
+	return bytes.Compare(a, b)
+}
+
+func (bytesComparer) Name() string {
+	return "leveldb.BytewiseComparator"
+}
+
+func (bytesComparer) Separator(dst, a, b []byte) []byte {
+	i, n := 0, len(a)
+	if n > len(b) {
+		n = len(b)
+	}
+	for ; i < n && a[i] == b[i]; i++ {
+	}
+	if i >= n {
+		// Do not shorten if one string is a prefix of the other
+	} else if c := a[i]; c < 0xff && c+1 < b[i] {
+		dst = append(dst, a[:i+1]...)
+		dst[i]++
+		return dst
+	}
+	return nil
+}
+
+func (bytesComparer) Successor(dst, b []byte) []byte {
+	for i, c := range b {
+		if c != 0xff {
+			dst = append(dst, b[:i+1]...)
+			dst[i]++
+			return dst
+		}
+	}
+	return nil
+}
+
+// DefaultComparer are default implementation of the Comparer interface.
+// It uses the natural ordering, consistent with bytes.Compare.
+var DefaultComparer = bytesComparer{}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package comparer provides interface and implementation for ordering
+// sets of data.
+package comparer
+
+// BasicComparer is the interface that wraps the basic Compare method.
+type BasicComparer interface {
+	// Compare returns -1, 0, or +1 depending on whether a is 'less than',
+	// 'equal to' or 'greater than' b. The two arguments can only be 'equal'
+	// if their contents are exactly equal. Furthermore, the empty slice
+	// must be 'less than' any non-empty slice.
+	Compare(a, b []byte) int
+}
+
+// Comparer defines a total ordering over the space of []byte keys: a 'less
+// than' relationship.
+type Comparer interface {
+	BasicComparer
+
+	// Name returns name of the comparer.
+	//
+	// The Level-DB on-disk format stores the comparer name, and opening a
+	// database with a different comparer from the one it was created with
+	// will result in an error.
+	//
+	// An implementation to a new name whenever the comparer implementation
+	// changes in a way that will cause the relative ordering of any two keys
+	// to change.
+	//
+	// Names starting with "leveldb." are reserved and should not be used
+	// by any users of this package.
+	Name() string
+
+	// Bellow are advanced functions used used to reduce the space requirements
+	// for internal data structures such as index blocks.
+
+	// Separator appends a sequence of bytes x to dst such that a <= x && x < b,
+	// where 'less than' is consistent with Compare. An implementation should
+	// return nil if x equal to a.
+	//
+	// Either contents of a or b should not by any means modified. Doing so
+	// may cause corruption on the internal state.
+	Separator(dst, a, b []byte) []byte
+
+	// Successor appends a sequence of bytes x to dst such that x >= b, where
+	// 'less than' is consistent with Compare. An implementation should return
+	// nil if x equal to b.
+	//
+	// Contents of b should not by any means modified. Doing so may cause
+	// corruption on the internal state.
+	Successor(dst, b []byte) []byte
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+const (
+	kNumLevels = 7
+
+	// Level-0 compaction is started when we hit this many files.
+	kL0_CompactionTrigger float64 = 4
+
+	// Soft limit on number of level-0 files.  We slow down writes at this point.
+	kL0_SlowdownWritesTrigger = 8
+
+	// Maximum number of level-0 files.  We stop writes at this point.
+	kL0_StopWritesTrigger = 12
+
+	// Maximum level to which a new compacted memdb is pushed if it
+	// does not create overlap.  We try to push to level 2 to avoid the
+	// relatively expensive level 0=>1 compactions and to avoid some
+	// expensive manifest file operations.  We do not push all the way to
+	// the largest level since that can generate a lot of wasted disk
+	// space if the same key space is being repeatedly overwritten.
+	kMaxMemCompactLevel = 2
+
+	// Maximum size of a table.
+	kMaxTableSize = 2 * 1048576
+
+	// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
+	// stop building a single file in a level->level+1 compaction.
+	kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
+
+	// Maximum number of bytes in all compacted files.  We avoid expanding
+	// the lower level file set of a compaction if it would make the
+	// total compaction cover more than this many bytes.
+	kExpCompactionMaxBytes = 25 * kMaxTableSize
+)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go

@@ -0,0 +1,472 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+const ctValSize = 1000
+
+type dbCorruptHarness struct {
+	dbHarness
+}
+
+func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
+	h := new(dbCorruptHarness)
+	h.init(t, o)
+	return h
+}
+
+func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
+	return newDbCorruptHarnessWopt(t, &opt.Options{
+		BlockCache: cache.NewLRUCache(100),
+		Strict:     opt.StrictJournalChecksum,
+	})
+}
+
+func (h *dbCorruptHarness) recover() {
+	p := &h.dbHarness
+	t := p.t
+
+	var err error
+	p.db, err = Recover(h.stor, h.o)
+	if err != nil {
+		t.Fatal("Repair: got error: ", err)
+	}
+}
+
+func (h *dbCorruptHarness) build(n int) {
+	p := &h.dbHarness
+	t := p.t
+	db := p.db
+
+	batch := new(Batch)
+	for i := 0; i < n; i++ {
+		batch.Reset()
+		batch.Put(tkey(i), tval(i, ctValSize))
+		err := db.Write(batch, p.wo)
+		if err != nil {
+			t.Fatal("write error: ", err)
+		}
+	}
+}
+
+func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) {
+	p := &h.dbHarness
+	t := p.t
+	db := p.db
+
+	batch := new(Batch)
+	for i := range rnd.Perm(n) {
+		batch.Reset()
+		batch.Put(tkey(i), tval(i, ctValSize))
+		err := db.Write(batch, p.wo)
+		if err != nil {
+			t.Fatal("write error: ", err)
+		}
+	}
+}
+
+func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
+	p := &h.dbHarness
+	t := p.t
+	db := p.db
+
+	batch := new(Batch)
+	for i := 0; i < n; i++ {
+		batch.Reset()
+		batch.Delete(tkey(rnd.Intn(max)))
+		err := db.Write(batch, p.wo)
+		if err != nil {
+			t.Fatal("write error: ", err)
+		}
+	}
+}
+
+func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) {
+	p := &h.dbHarness
+	t := p.t
+
+	var file storage.File
+	ff, _ := p.stor.GetFiles(ft)
+	for _, f := range ff {
+		if file == nil || f.Num() > file.Num() {
+			file = f
+		}
+	}
+	if file == nil {
+		t.Fatalf("no such file with type %q", ft)
+	}
+
+	r, err := file.Open()
+	if err != nil {
+		t.Fatal("cannot open file: ", err)
+	}
+	x, err := r.Seek(0, 2)
+	if err != nil {
+		t.Fatal("cannot query file size: ", err)
+	}
+	m := int(x)
+	if _, err := r.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	if offset < 0 {
+		if -offset > m {
+			offset = 0
+		} else {
+			offset = m + offset
+		}
+	}
+	if offset > m {
+		offset = m
+	}
+	if offset+n > m {
+		n = m - offset
+	}
+
+	buf := make([]byte, m)
+	_, err = io.ReadFull(r, buf)
+	if err != nil {
+		t.Fatal("cannot read file: ", err)
+	}
+	r.Close()
+
+	for i := 0; i < n; i++ {
+		buf[offset+i] ^= 0x80
+	}
+
+	err = file.Remove()
+	if err != nil {
+		t.Fatal("cannot remove old file: ", err)
+	}
+	w, err := file.Create()
+	if err != nil {
+		t.Fatal("cannot create new file: ", err)
+	}
+	_, err = w.Write(buf)
+	if err != nil {
+		t.Fatal("cannot write new file: ", err)
+	}
+	w.Close()
+}
+
+func (h *dbCorruptHarness) removeAll(ft storage.FileType) {
+	ff, err := h.stor.GetFiles(ft)
+	if err != nil {
+		h.t.Fatal("get files: ", err)
+	}
+	for _, f := range ff {
+		if err := f.Remove(); err != nil {
+			h.t.Error("remove file: ", err)
+		}
+	}
+}
+
+func (h *dbCorruptHarness) removeOne(ft storage.FileType) {
+	ff, err := h.stor.GetFiles(ft)
+	if err != nil {
+		h.t.Fatal("get files: ", err)
+	}
+	f := ff[rand.Intn(len(ff))]
+	h.t.Logf("removing file @%d", f.Num())
+	if err := f.Remove(); err != nil {
+		h.t.Error("remove file: ", err)
+	}
+}
+
+func (h *dbCorruptHarness) check(min, max int) {
+	p := &h.dbHarness
+	t := p.t
+	db := p.db
+
+	var n, badk, badv, missed, good int
+	iter := db.NewIterator(nil, p.ro)
+	for iter.Next() {
+		k := 0
+		fmt.Sscanf(string(iter.Key()), "%d", &k)
+		if k < n {
+			badk++
+			continue
+		}
+		missed += k - n
+		n = k + 1
+		if !bytes.Equal(iter.Value(), tval(k, ctValSize)) {
+			badv++
+		} else {
+			good++
+		}
+	}
+	err := iter.Error()
+	iter.Release()
+	t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v",
+		min, max, good, badk, badv, missed, err)
+	if good < min || good > max {
+		t.Errorf("good entries number not in range")
+	}
+}
+
+func TestCorruptDB_Journal(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.build(100)
+	h.check(100, 100)
+	h.closeDB()
+	h.corrupt(storage.TypeJournal, 19, 1)
+	h.corrupt(storage.TypeJournal, 32*1024+1000, 1)
+
+	h.openDB()
+	h.check(36, 36)
+
+	h.close()
+}
+
+func TestCorruptDB_Table(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.build(100)
+	h.compactMem()
+	h.compactRangeAt(0, "", "")
+	h.compactRangeAt(1, "", "")
+	h.closeDB()
+	h.corrupt(storage.TypeTable, 100, 1)
+
+	h.openDB()
+	h.check(99, 99)
+
+	h.close()
+}
+
+func TestCorruptDB_TableIndex(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.build(10000)
+	h.compactMem()
+	h.closeDB()
+	h.corrupt(storage.TypeTable, -2000, 500)
+
+	h.openDB()
+	h.check(5000, 9999)
+
+	h.close()
+}
+
+func TestCorruptDB_MissingManifest(t *testing.T) {
+	rnd := rand.New(rand.NewSource(0x0badda7a))
+	h := newDbCorruptHarnessWopt(t, &opt.Options{
+		BlockCache:  cache.NewLRUCache(100),
+		Strict:      opt.StrictJournalChecksum,
+		WriteBuffer: 1000 * 60,
+	})
+
+	h.build(1000)
+	h.compactMem()
+	h.buildShuffled(1000, rnd)
+	h.compactMem()
+	h.deleteRand(500, 1000, rnd)
+	h.compactMem()
+	h.buildShuffled(1000, rnd)
+	h.compactMem()
+	h.deleteRand(500, 1000, rnd)
+	h.compactMem()
+	h.buildShuffled(1000, rnd)
+	h.compactMem()
+	h.closeDB()
+
+	h.stor.SetIgnoreOpenErr(storage.TypeManifest)
+	h.removeAll(storage.TypeManifest)
+	h.openAssert(false)
+	h.stor.SetIgnoreOpenErr(0)
+
+	h.recover()
+	h.check(1000, 1000)
+	h.build(1000)
+	h.compactMem()
+	h.compactRange("", "")
+	h.closeDB()
+
+	h.recover()
+	h.check(1000, 1000)
+
+	h.close()
+}
+
+func TestCorruptDB_SequenceNumberRecovery(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("foo", "v1")
+	h.put("foo", "v2")
+	h.put("foo", "v3")
+	h.put("foo", "v4")
+	h.put("foo", "v5")
+	h.closeDB()
+
+	h.recover()
+	h.getVal("foo", "v5")
+	h.put("foo", "v6")
+	h.getVal("foo", "v6")
+
+	h.reopenDB()
+	h.getVal("foo", "v6")
+
+	h.close()
+}
+
+func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("foo", "v1")
+	h.put("foo", "v2")
+	h.put("foo", "v3")
+	h.compactMem()
+	h.put("foo", "v4")
+	h.put("foo", "v5")
+	h.compactMem()
+	h.closeDB()
+
+	h.recover()
+	h.getVal("foo", "v5")
+	h.put("foo", "v6")
+	h.getVal("foo", "v6")
+
+	h.reopenDB()
+	h.getVal("foo", "v6")
+
+	h.close()
+}
+
+func TestCorruptDB_CorruptedManifest(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("foo", "hello")
+	h.compactMem()
+	h.compactRange("", "")
+	h.closeDB()
+	h.corrupt(storage.TypeManifest, 0, 1000)
+	h.openAssert(false)
+
+	h.recover()
+	h.getVal("foo", "hello")
+
+	h.close()
+}
+
+func TestCorruptDB_CompactionInputError(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.build(10)
+	h.compactMem()
+	h.closeDB()
+	h.corrupt(storage.TypeTable, 100, 1)
+
+	h.openDB()
+	h.check(9, 9)
+
+	h.build(10000)
+	h.check(10000, 10000)
+
+	h.close()
+}
+
+func TestCorruptDB_UnrelatedKeys(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.build(10)
+	h.compactMem()
+	h.closeDB()
+	h.corrupt(storage.TypeTable, 100, 1)
+
+	h.openDB()
+	h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
+	h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
+	h.compactMem()
+	h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
+
+	h.close()
+}
+
+func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("a", "v1")
+	h.put("b", "v1")
+	h.compactMem()
+	h.put("a", "v2")
+	h.put("b", "v2")
+	h.compactMem()
+	h.put("a", "v3")
+	h.put("b", "v3")
+	h.compactMem()
+	h.put("c", "v0")
+	h.put("d", "v0")
+	h.compactMem()
+	h.compactRangeAt(1, "", "")
+	h.closeDB()
+
+	h.recover()
+	h.getVal("a", "v3")
+	h.getVal("b", "v3")
+	h.getVal("c", "v0")
+	h.getVal("d", "v0")
+
+	h.close()
+}
+
+func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("a", "v1")
+	h.put("b", "v1")
+	h.compactMem()
+	h.put("a", "v2")
+	h.put("b", "v2")
+	h.compactMem()
+	h.put("a", "v3")
+	h.put("b", "v3")
+	h.compactMem()
+	h.put("c", "v0")
+	h.put("d", "v0")
+	h.compactMem()
+	h.compactRangeAt(0, "", "")
+	h.closeDB()
+
+	h.recover()
+	h.getVal("a", "v3")
+	h.getVal("b", "v3")
+	h.getVal("c", "v0")
+	h.getVal("d", "v0")
+
+	h.close()
+}
+
+func TestCorruptDB_MissingTableFiles(t *testing.T) {
+	h := newDbCorruptHarness(t)
+
+	h.put("a", "v1")
+	h.put("b", "v1")
+	h.compactMem()
+	h.put("c", "v2")
+	h.put("d", "v2")
+	h.compactMem()
+	h.put("e", "v3")
+	h.put("f", "v3")
+	h.closeDB()
+
+	h.removeOne(storage.TypeTable)
+	h.openAssert(false)
+
+	h.close()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -0,0 +1,752 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/journal"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/table"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// DB is a LevelDB database.
+type DB struct {
+	// Need 64-bit alignment.
+	seq uint64
+
+	s *session
+
+	// MemDB
+	memMu             sync.RWMutex
+	mem               *memdb.DB
+	frozenMem         *memdb.DB
+	journal           *journal.Writer
+	journalWriter     storage.Writer
+	journalFile       storage.File
+	frozenJournalFile storage.File
+	frozenSeq         uint64
+
+	// Snapshot
+	snapsMu   sync.Mutex
+	snapsRoot snapshotElement
+
+	// Write
+	writeC       chan *Batch
+	writeMergedC chan bool
+	writeLockC   chan struct{}
+	writeAckC    chan error
+	journalC     chan *Batch
+	journalAckC  chan error
+
+	// Compaction
+	tcompCmdC     chan cCmd
+	tcompPauseC   chan chan<- struct{}
+	tcompTriggerC chan struct{}
+	mcompCmdC     chan cCmd
+	mcompTriggerC chan struct{}
+	compErrC      chan error
+	compErrSetC   chan error
+	compStats     [kNumLevels]cStats
+
+	// Close
+	closeW sync.WaitGroup
+	closeC chan struct{}
+	closed uint32
+	closer io.Closer
+}
+
+func openDB(s *session) (*DB, error) {
+	s.log("db@open opening")
+	start := time.Now()
+	db := &DB{
+		s: s,
+		// Initial sequence
+		seq: s.stSeq,
+		// Write
+		writeC:       make(chan *Batch),
+		writeMergedC: make(chan bool),
+		writeLockC:   make(chan struct{}, 1),
+		writeAckC:    make(chan error),
+		journalC:     make(chan *Batch),
+		journalAckC:  make(chan error),
+		// Compaction
+		tcompCmdC:     make(chan cCmd),
+		tcompPauseC:   make(chan chan<- struct{}),
+		tcompTriggerC: make(chan struct{}, 1),
+		mcompCmdC:     make(chan cCmd),
+		mcompTriggerC: make(chan struct{}, 1),
+		compErrC:      make(chan error),
+		compErrSetC:   make(chan error),
+		// Close
+		closeC: make(chan struct{}),
+	}
+	db.initSnapshot()
+
+	if err := db.recoverJournal(); err != nil {
+		return nil, err
+	}
+
+	// Remove any obsolete files.
+	if err := db.checkAndCleanFiles(); err != nil {
+		// Close journal.
+		if db.journal != nil {
+			db.journal.Close()
+			db.journalWriter.Close()
+		}
+		return nil, err
+	}
+
+	// Don't include compaction error goroutine into wait group.
+	go db.compactionError()
+
+	db.closeW.Add(3)
+	go db.tCompaction()
+	go db.mCompaction()
+	go db.jWriter()
+
+	s.logf("db@open done T·%v", time.Since(start))
+
+	runtime.SetFinalizer(db, (*DB).Close)
+	return db, nil
+}
+
+// Open opens or creates a DB for the given storage.
+// The DB will be created if not exist, unless ErrorIfMissing is true.
+// Also, if ErrorIfExist is true and the DB exist Open will returns
+// os.ErrExist error.
+//
+// Open will return an error with type of ErrCorrupted if corruption
+// detected in the DB. Corrupted DB can be recovered with Recover
+// function.
+//
+// The DB must be closed after use, by calling Close method.
+func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
+	s, err := newSession(p, o)
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			s.close()
+			s.release()
+		}
+	}()
+
+	err = s.recover()
+	if err != nil {
+		if !os.IsNotExist(err) || s.o.GetErrorIfMissing() {
+			return
+		}
+		err = s.create()
+		if err != nil {
+			return
+		}
+	} else if s.o.GetErrorIfExist() {
+		err = os.ErrExist
+		return
+	}
+
+	return openDB(s)
+}
+
+// OpenFile opens or creates a DB for the given path.
+// The DB will be created if not exist, unless ErrorIfMissing is true.
+// Also, if ErrorIfExist is true and the DB exist OpenFile will returns
+// os.ErrExist error.
+//
+// OpenFile uses standard file-system backed storage implementation as
+// desribed in the leveldb/storage package.
+//
+// OpenFile will return an error with type of ErrCorrupted if corruption
+// detected in the DB. Corrupted DB can be recovered with Recover
+// function.
+//
+// The DB must be closed after use, by calling Close method.
+func OpenFile(path string, o *opt.Options) (db *DB, err error) {
+	stor, err := storage.OpenFile(path)
+	if err != nil {
+		return
+	}
+	db, err = Open(stor, o)
+	if err != nil {
+		stor.Close()
+	} else {
+		db.closer = stor
+	}
+	return
+}
+
+// Recover recovers and opens a DB with missing or corrupted manifest files
+// for the given storage. It will ignore any manifest files, valid or not.
+// The DB must already exist or it will returns an error.
+// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
+//
+// The DB must be closed after use, by calling Close method.
+func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
+	s, err := newSession(p, o)
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			s.close()
+			s.release()
+		}
+	}()
+
+	err = recoverTable(s, o)
+	if err != nil {
+		return
+	}
+	return openDB(s)
+}
+
+// RecoverFile recovers and opens a DB with missing or corrupted manifest files
+// for the given path. It will ignore any manifest files, valid or not.
+// The DB must already exist or it will returns an error.
+// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
+//
+// RecoverFile uses standard file-system backed storage implementation as desribed
+// in the leveldb/storage package.
+//
+// The DB must be closed after use, by calling Close method.
+func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
+	stor, err := storage.OpenFile(path)
+	if err != nil {
+		return
+	}
+	db, err = Recover(stor, o)
+	if err != nil {
+		stor.Close()
+	} else {
+		db.closer = stor
+	}
+	return
+}
+
+func recoverTable(s *session, o *opt.Options) error {
+	ff0, err := s.getFiles(storage.TypeTable)
+	if err != nil {
+		return err
+	}
+	ff1 := files(ff0)
+	ff1.sort()
+
+	var mSeq uint64
+	var good, corrupted int
+	rec := new(sessionRecord)
+	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
+		tmp = s.newTemp()
+		writer, err := tmp.Create()
+		if err != nil {
+			return
+		}
+		defer func() {
+			writer.Close()
+			if err != nil {
+				tmp.Remove()
+				tmp = nil
+			}
+		}()
+		tw := table.NewWriter(writer, o)
+		// Copy records.
+		for iter.Next() {
+			key := iter.Key()
+			if validIkey(key) {
+				err = tw.Append(key, iter.Value())
+				if err != nil {
+					return
+				}
+			}
+		}
+		err = iter.Error()
+		if err != nil {
+			return
+		}
+		err = tw.Close()
+		if err != nil {
+			return
+		}
+		err = writer.Sync()
+		if err != nil {
+			return
+		}
+		size = int64(tw.BytesLen())
+		return
+	}
+	recoverTable := func(file storage.File) error {
+		s.logf("table@recovery recovering @%d", file.Num())
+		reader, err := file.Open()
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+		// Get file size.
+		size, err := reader.Seek(0, 2)
+		if err != nil {
+			return err
+		}
+		var tSeq uint64
+		var tgood, tcorrupted, blockerr int
+		var min, max []byte
+		tr := table.NewReader(reader, size, nil, o)
+		iter := tr.NewIterator(nil, nil)
+		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
+			s.logf("table@recovery found error @%d %q", file.Num(), err)
+			blockerr++
+		})
+		// Scan the table.
+		for iter.Next() {
+			key := iter.Key()
+			_, seq, _, ok := parseIkey(key)
+			if !ok {
+				tcorrupted++
+				continue
+			}
+			tgood++
+			if seq > tSeq {
+				tSeq = seq
+			}
+			if min == nil {
+				min = append([]byte{}, key...)
+			}
+			max = append(max[:0], key...)
+		}
+		if err := iter.Error(); err != nil {
+			iter.Release()
+			return err
+		}
+		iter.Release()
+		if tgood > 0 {
+			if tcorrupted > 0 || blockerr > 0 {
+				// Rebuild the table.
+				s.logf("table@recovery rebuilding @%d", file.Num())
+				iter := tr.NewIterator(nil, nil)
+				tmp, newSize, err := buildTable(iter)
+				iter.Release()
+				if err != nil {
+					return err
+				}
+				reader.Close()
+				if err := file.Replace(tmp); err != nil {
+					return err
+				}
+				size = newSize
+			}
+			if tSeq > mSeq {
+				mSeq = tSeq
+			}
+			// Add table to level 0.
+			rec.addTable(0, file.Num(), uint64(size), min, max)
+			s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
+		} else {
+			s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
+		}
+
+		good += tgood
+		corrupted += tcorrupted
+
+		return nil
+	}
+	// Recover all tables.
+	if len(ff1) > 0 {
+		s.logf("table@recovery F·%d", len(ff1))
+		s.markFileNum(ff1[len(ff1)-1].Num())
+		for _, file := range ff1 {
+			if err := recoverTable(file); err != nil {
+				return err
+			}
+		}
+		s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(ff1), good, corrupted, mSeq)
+	}
+	// Set sequence number.
+	rec.setSeq(mSeq + 1)
+	// Create new manifest.
+	if err := s.create(); err != nil {
+		return err
+	}
+	// Commit.
+	return s.commit(rec)
+}
+
+func (d *DB) recoverJournal() error {
+	s := d.s
+	icmp := s.cmp
+
+	ff0, err := s.getFiles(storage.TypeJournal)
+	if err != nil {
+		return err
+	}
+	ff1 := files(ff0)
+	ff1.sort()
+	ff2 := make([]storage.File, 0, len(ff1))
+	for _, file := range ff1 {
+		if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
+			s.markFileNum(file.Num())
+			ff2 = append(ff2, file)
+		}
+	}
+
+	var jr *journal.Reader
+	var of storage.File
+	var mem *memdb.DB
+	batch := new(Batch)
+	cm := newCMem(s)
+	buf := new(util.Buffer)
+	// Options.
+	strict := s.o.GetStrict(opt.StrictJournal)
+	checksum := s.o.GetStrict(opt.StrictJournalChecksum)
+	writeBuffer := s.o.GetWriteBuffer()
+	recoverJournal := func(file storage.File) error {
+		s.logf("journal@recovery recovering @%d", file.Num())
+		reader, err := file.Open()
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+		if jr == nil {
+			jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
+		} else {
+			jr.Reset(reader, dropper{s, file}, strict, checksum)
+		}
+		if of != nil {
+			if mem.Len() > 0 {
+				if err := cm.flush(mem, 0); err != nil {
+					return err
+				}
+			}
+			if err := cm.commit(file.Num(), d.seq); err != nil {
+				return err
+			}
+			cm.reset()
+			of.Remove()
+			of = nil
+		}
+		// Reset memdb.
+		mem.Reset()
+		for {
+			r, err := jr.Next()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return err
+			}
+			buf.Reset()
+			if _, err := buf.ReadFrom(r); err != nil {
+				if strict {
+					return err
+				}
+				continue
+			}
+			if err := batch.decode(buf.Bytes()); err != nil {
+				return err
+			}
+			if err := batch.memReplay(mem); err != nil {
+				return err
+			}
+			d.seq = batch.seq + uint64(batch.len())
+			if mem.Size() >= writeBuffer {
+				// Large enough, flush it.
+				if err := cm.flush(mem, 0); err != nil {
+					return err
+				}
+				// Reset memdb.
+				mem.Reset()
+			}
+		}
+		of = file
+		return nil
+	}
+	// Recover all journals.
+	if len(ff2) > 0 {
+		s.logf("journal@recovery F·%d", len(ff2))
+		mem = memdb.New(icmp, writeBuffer)
+		for _, file := range ff2 {
+			if err := recoverJournal(file); err != nil {
+				return err
+			}
+		}
+		// Flush the last journal.
+		if mem.Len() > 0 {
+			if err := cm.flush(mem, 0); err != nil {
+				return err
+			}
+		}
+	}
+	// Create a new journal.
+	if _, err := d.newMem(0); err != nil {
+		return err
+	}
+	// Commit.
+	if err := cm.commit(d.journalFile.Num(), d.seq); err != nil {
+		return err
+	}
+	// Remove the last journal.
+	if of != nil {
+		of.Remove()
+	}
+	return nil
+}
+
+func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
+	s := d.s
+
+	ucmp := s.cmp.cmp
+	ikey := newIKey(key, seq, tSeek)
+
+	em, fm := d.getMems()
+	for _, m := range [...]*memdb.DB{em, fm} {
+		if m == nil {
+			continue
+		}
+		mk, mv, me := m.Find(ikey)
+		if me == nil {
+			ukey, _, t, ok := parseIkey(mk)
+			if ok && ucmp.Compare(ukey, key) == 0 {
+				if t == tDel {
+					return nil, ErrNotFound
+				}
+				return mv, nil
+			}
+		} else if me != ErrNotFound {
+			return nil, me
+		}
+	}
+
+	v := s.version()
+	value, cSched, err := v.get(ikey, ro)
+	v.release()
+	if cSched {
+		// Trigger table compaction.
+		d.compTrigger(d.tcompTriggerC)
+	}
+	return
+}
+
+// Get gets the value for the given key. It returns ErrNotFound if the
+// DB does not contain the key.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Get returns.
+func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	err = d.ok()
+	if err != nil {
+		return
+	}
+
+	return d.get(key, d.getSeq(), ro)
+}
+
+// NewIterator returns an iterator for the latest snapshot of the
+// uderlying DB.
+// The returned iterator is not goroutine-safe, but it is safe to use
+// multiple iterators concurrently, with each in a dedicated goroutine.
+// It is also safe to use an iterator concurrently with modifying its
+// underlying DB. The resultant key/value pairs are guaranteed to be
+// consistent.
+//
+// Slice allows slicing the iterator to only contains keys in the given
+// range. A nil Range.Start is treated as a key before all keys in the
+// DB. And a nil Range.Limit is treated as a key after all keys in
+// the DB.
+//
+// The iterator must be released after use, by calling Release method.
+//
+// Also read Iterator documentation of the leveldb/iterator package.
+func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	if err := d.ok(); err != nil {
+		return iterator.NewEmptyIterator(err)
+	}
+
+	p := d.newSnapshot()
+	defer p.Release()
+	return p.NewIterator(slice, ro)
+}
+
+// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
+// is a frozen snapshot of a DB state at a particular point in time. The
+// content of snapshot are guaranteed to be consistent.
+//
+// The snapshot must be released after use, by calling Release method.
+func (d *DB) GetSnapshot() (*Snapshot, error) {
+	if err := d.ok(); err != nil {
+		return nil, err
+	}
+
+	return d.newSnapshot(), nil
+}
+
+// GetProperty returns value of the given property name.
+//
+// Property names:
+//	leveldb.num-files-at-level{n}
+//		Returns the number of filer at level 'n'.
+//	leveldb.stats
+//		Returns statistics of the underlying DB.
+//	leveldb.sstables
+//		Returns sstables list for each level.
+func (d *DB) GetProperty(name string) (value string, err error) {
+	err = d.ok()
+	if err != nil {
+		return
+	}
+
+	const prefix = "leveldb."
+	if !strings.HasPrefix(name, prefix) {
+		return "", errors.New("leveldb: GetProperty: unknown property: " + name)
+	}
+
+	p := name[len(prefix):]
+
+	s := d.s
+	v := s.version()
+	defer v.release()
+
+	switch {
+	case strings.HasPrefix(p, "num-files-at-level"):
+		var level uint
+		var rest string
+		n, _ := fmt.Scanf("%d%s", &level, &rest)
+		if n != 1 || level >= kNumLevels {
+			err = errors.New("leveldb: GetProperty: invalid property: " + name)
+		} else {
+			value = fmt.Sprint(v.tLen(int(level)))
+		}
+	case p == "stats":
+		value = "Compactions\n" +
+			" Level |   Tables   |    Size(MB)   |    Time(sec)  |    Read(MB)   |   Write(MB)\n" +
+			"-------+------------+---------------+---------------+---------------+---------------\n"
+		for level, tt := range v.tables {
+			duration, read, write := d.compStats[level].get()
+			if len(tt) == 0 && duration == 0 {
+				continue
+			}
+			value += fmt.Sprintf(" %3d   | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
+				level, len(tt), float64(tt.size())/1048576.0, duration.Seconds(),
+				float64(read)/1048576.0, float64(write)/1048576.0)
+		}
+	case p == "sstables":
+		for level, tt := range v.tables {
+			value += fmt.Sprintf("--- level %d ---\n", level)
+			for _, t := range tt {
+				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.min, t.max)
+			}
+		}
+	default:
+		err = errors.New("leveldb: GetProperty: unknown property: " + name)
+	}
+
+	return
+}
+
+// GetApproximateSizes calculates approximate sizes of the given key ranges.
+// The length of the returned sizes are equal with the length of the given
+// ranges. The returned sizes measure storage space usage, so if the user
+// data compresses by a factor of ten, the returned sizes will be one-tenth
+// the size of the corresponding user data size.
+// The results may not include the sizes of recently written data.
+func (d *DB) GetApproximateSizes(ranges []util.Range) (Sizes, error) {
+	if err := d.ok(); err != nil {
+		return nil, err
+	}
+
+	v := d.s.version()
+	defer v.release()
+
+	sizes := make(Sizes, 0, len(ranges))
+	for _, r := range ranges {
+		min := newIKey(r.Start, kMaxSeq, tSeek)
+		max := newIKey(r.Limit, kMaxSeq, tSeek)
+		start, err := v.getApproximateOffset(min)
+		if err != nil {
+			return nil, err
+		}
+		limit, err := v.getApproximateOffset(max)
+		if err != nil {
+			return nil, err
+		}
+		var size uint64
+		if limit >= start {
+			size = limit - start
+		}
+		sizes = append(sizes, size)
+	}
+
+	return sizes, nil
+}
+
+// Close closes the DB. This will also releases any outstanding snapshot.
+//
+// It is not safe to close a DB until all outstanding iterators are released.
+// It is valid to call Close multiple times. Other methods should not be
+// called after the DB has been closed.
+func (d *DB) Close() error {
+	if !d.setClosed() {
+		return ErrClosed
+	}
+
+	s := d.s
+	start := time.Now()
+	s.log("db@close closing")
+
+	// Clear the finalizer.
+	runtime.SetFinalizer(d, nil)
+
+	// Get compaction error.
+	var err error
+	select {
+	case err = <-d.compErrC:
+	default:
+	}
+
+	close(d.closeC)
+
+	// Wait for the close WaitGroup.
+	d.closeW.Wait()
+
+	// Close journal.
+	if d.journal != nil {
+		d.journal.Close()
+		d.journalWriter.Close()
+	}
+
+	// Close session.
+	s.close()
+	s.logf("db@close done T·%v", time.Since(start))
+	s.release()
+
+	if d.closer != nil {
+		if err1 := d.closer.Close(); err == nil {
+			err = err1
+		}
+	}
+
+	d.s = nil
+	d.mem = nil
+	d.frozenMem = nil
+	d.journal = nil
+	d.journalWriter = nil
+	d.journalFile = nil
+	d.frozenJournalFile = nil
+	d.snapsRoot = snapshotElement{}
+	d.closer = nil
+
+	return err
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -0,0 +1,689 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+)
+
+var (
+	errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting")
+)
+
+type cStats struct {
+	sync.Mutex
+	duration time.Duration
+	read     uint64
+	write    uint64
+}
+
+func (p *cStats) add(n *cStatsStaging) {
+	p.Lock()
+	p.duration += n.duration
+	p.read += n.read
+	p.write += n.write
+	p.Unlock()
+}
+
+func (p *cStats) get() (duration time.Duration, read, write uint64) {
+	p.Lock()
+	defer p.Unlock()
+	return p.duration, p.read, p.write
+}
+
+type cStatsStaging struct {
+	start    time.Time
+	duration time.Duration
+	on       bool
+	read     uint64
+	write    uint64
+}
+
+func (p *cStatsStaging) startTimer() {
+	if !p.on {
+		p.start = time.Now()
+		p.on = true
+	}
+}
+
+func (p *cStatsStaging) stopTimer() {
+	if p.on {
+		p.duration += time.Since(p.start)
+		p.on = false
+	}
+}
+
+type cMem struct {
+	s     *session
+	level int
+	rec   *sessionRecord
+}
+
+func newCMem(s *session) *cMem {
+	return &cMem{s: s, rec: new(sessionRecord)}
+}
+
+func (c *cMem) flush(mem *memdb.DB, level int) error {
+	s := c.s
+
+	// Write memdb to table
+	iter := mem.NewIterator(nil)
+	defer iter.Release()
+	t, n, err := s.tops.createFrom(iter)
+	if err != nil {
+		return err
+	}
+
+	if level < 0 {
+		level = s.version_NB().pickLevel(t.min.ukey(), t.max.ukey())
+	}
+	c.rec.addTableFile(level, t)
+
+	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.min, t.max)
+
+	c.level = level
+	return nil
+}
+
+func (c *cMem) reset() {
+	c.rec = new(sessionRecord)
+}
+
+func (c *cMem) commit(journal, seq uint64) error {
+	c.rec.setJournalNum(journal)
+	c.rec.setSeq(seq)
+	// Commit changes
+	return c.s.commit(c.rec)
+}
+
+func (d *DB) compactionError() {
+	var err error
+noerr:
+	for {
+		select {
+		case _, _ = <-d.closeC:
+			return
+		case err = <-d.compErrSetC:
+			if err != nil {
+				goto haserr
+			}
+		}
+	}
+haserr:
+	for {
+		select {
+		case _, _ = <-d.closeC:
+			return
+		case err = <-d.compErrSetC:
+			if err == nil {
+				goto noerr
+			}
+		case d.compErrC <- err:
+		}
+	}
+}
+
+type compactionTransactCounter int
+
+func (cnt *compactionTransactCounter) incr() {
+	*cnt++
+}
+
+func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
+	s := d.s
+	defer func() {
+		if x := recover(); x != nil {
+			if x == errCompactionTransactExiting && rollback != nil {
+				if err := rollback(); err != nil {
+					s.logf("%s rollback error %q", name, err)
+				}
+			}
+			panic(x)
+		}
+	}()
+	const (
+		backoffMin = 1 * time.Second
+		backoffMax = 8 * time.Second
+		backoffMul = 2 * time.Second
+	)
+	backoff := backoffMin
+	backoffT := time.NewTimer(backoff)
+	lastCnt := compactionTransactCounter(0)
+	for n := 0; ; n++ {
+		// Check wether the DB is closed.
+		if d.isClosed() {
+			s.logf("%s exiting", name)
+			d.compactionExitTransact()
+		} else if n > 0 {
+			s.logf("%s retrying N·%d", name, n)
+		}
+
+		// Execute.
+		cnt := compactionTransactCounter(0)
+		err := exec(&cnt)
+
+		// Set compaction error status.
+		select {
+		case d.compErrSetC <- err:
+		case _, _ = <-d.closeC:
+			s.logf("%s exiting", name)
+			d.compactionExitTransact()
+		}
+		if err == nil {
+			return
+		}
+		s.logf("%s error I·%d %q", name, cnt, err)
+
+		// Reset backoff duration if counter is advancing.
+		if cnt > lastCnt {
+			backoff = backoffMin
+			lastCnt = cnt
+		}
+
+		// Backoff.
+		backoffT.Reset(backoff)
+		if backoff < backoffMax {
+			backoff *= backoffMul
+			if backoff > backoffMax {
+				backoff = backoffMax
+			}
+		}
+		select {
+		case <-backoffT.C:
+		case _, _ = <-d.closeC:
+			s.logf("%s exiting", name)
+			d.compactionExitTransact()
+		}
+	}
+}
+
+func (d *DB) compactionExitTransact() {
+	panic(errCompactionTransactExiting)
+}
+
+func (d *DB) memCompaction() {
+	mem := d.getFrozenMem()
+	if mem == nil {
+		return
+	}
+
+	s := d.s
+	c := newCMem(s)
+	stats := new(cStatsStaging)
+
+	s.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
+
+	// Don't compact empty memdb.
+	if mem.Len() == 0 {
+		s.logf("mem@flush skipping")
+		// drop frozen mem
+		d.dropFrozenMem()
+		return
+	}
+
+	// Pause table compaction.
+	ch := make(chan struct{})
+	select {
+	case d.tcompPauseC <- ch:
+	case _, _ = <-d.closeC:
+		return
+	}
+
+	d.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
+		stats.startTimer()
+		defer stats.stopTimer()
+		return c.flush(mem, -1)
+	}, func() error {
+		for _, r := range c.rec.addedTables {
+			s.logf("mem@flush rollback @%d", r.num)
+			f := s.getTableFile(r.num)
+			if err := f.Remove(); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	d.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
+		stats.startTimer()
+		defer stats.stopTimer()
+		return c.commit(d.journalFile.Num(), d.frozenSeq)
+	}, nil)
+
+	s.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
+
+	for _, r := range c.rec.addedTables {
+		stats.write += r.size
+	}
+	d.compStats[c.level].add(stats)
+
+	// Drop frozen mem.
+	d.dropFrozenMem()
+
+	// Unpause table compaction.
+	select {
+	case <-ch:
+	case _, _ = <-d.closeC:
+		return
+	}
+
+	// Trigger table compaction.
+	d.compTrigger(d.mcompTriggerC)
+}
+
+func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
+	s := d.s
+	ucmp := s.cmp.cmp
+
+	rec := new(sessionRecord)
+	rec.addCompactionPointer(c.level, c.max)
+
+	if !noTrivial && c.trivial() {
+		t := c.tables[0][0]
+		s.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
+		rec.deleteTable(c.level, t.file.Num())
+		rec.addTableFile(c.level+1, t)
+		d.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
+			return s.commit(rec)
+		}, nil)
+		return
+	}
+
+	var stats [2]cStatsStaging
+	for i, tt := range c.tables {
+		for _, t := range tt {
+			stats[i].read += t.size
+			// Insert deleted tables into record
+			rec.deleteTable(c.level+i, t.file.Num())
+		}
+	}
+	sourceSize := int(stats[0].read + stats[1].read)
+	minSeq := d.minSeq()
+	s.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
+
+	var snapUkey []byte
+	var snapHasUkey bool
+	var snapSeq uint64
+	var snapIter int
+	var snapDropCnt int
+	var dropCnt int
+	d.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
+		ukey := append([]byte{}, snapUkey...)
+		hasUkey := snapHasUkey
+		lseq := snapSeq
+		dropCnt = snapDropCnt
+		snapSched := snapIter == 0
+
+		var tw *tWriter
+		finish := func() error {
+			t, err := tw.finish()
+			if err != nil {
+				return err
+			}
+			rec.addTableFile(c.level+1, t)
+			stats[1].write += t.size
+			s.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.min, t.max)
+			return nil
+		}
+
+		defer func() {
+			stats[1].stopTimer()
+			if tw != nil {
+				tw.drop()
+				tw = nil
+			}
+		}()
+
+		stats[1].startTimer()
+		iter := c.newIterator()
+		defer iter.Release()
+		for i := 0; iter.Next(); i++ {
+			// Incr transact counter.
+			cnt.incr()
+
+			// Skip until last state.
+			if i < snapIter {
+				continue
+			}
+
+			key := iKey(iter.Key())
+
+			if c.shouldStopBefore(key) && tw != nil {
+				err = finish()
+				if err != nil {
+					return
+				}
+				snapSched = true
+				tw = nil
+			}
+
+			// Scheduled for snapshot, snapshot will used to retry compaction
+			// if error occured.
+			if snapSched {
+				snapUkey = append(snapUkey[:0], ukey...)
+				snapHasUkey = hasUkey
+				snapSeq = lseq
+				snapIter = i
+				snapDropCnt = dropCnt
+				snapSched = false
+			}
+
+			if seq, t, ok := key.parseNum(); !ok {
+				// Don't drop error keys
+				ukey = ukey[:0]
+				hasUkey = false
+				lseq = kMaxSeq
+			} else {
+				if !hasUkey || ucmp.Compare(key.ukey(), ukey) != 0 {
+					// First occurrence of this user key
+					ukey = append(ukey[:0], key.ukey()...)
+					hasUkey = true
+					lseq = kMaxSeq
+				}
+
+				drop := false
+				if lseq <= minSeq {
+					// Dropped because newer entry for same user key exist
+					drop = true // (A)
+				} else if t == tDel && seq <= minSeq && c.isBaseLevelForKey(ukey) {
+					// For this user key:
+					// (1) there is no data in higher levels
+					// (2) data in lower levels will have larger seq numbers
+					// (3) data in layers that are being compacted here and have
+					//     smaller seq numbers will be dropped in the next
+					//     few iterations of this loop (by rule (A) above).
+					// Therefore this deletion marker is obsolete and can be dropped.
+					drop = true
+				}
+
+				lseq = seq
+				if drop {
+					dropCnt++
+					continue
+				}
+			}
+
+			// Create new table if not already
+			if tw == nil {
+				// Check for pause event.
+				select {
+				case ch := <-d.tcompPauseC:
+					d.pauseCompaction(ch)
+				case _, _ = <-d.closeC:
+					d.compactionExitTransact()
+				default:
+				}
+
+				// Create new table.
+				tw, err = s.tops.create()
+				if err != nil {
+					return
+				}
+			}
+
+			// Write key/value into table
+			err = tw.add(key, iter.Value())
+			if err != nil {
+				return
+			}
+
+			// Finish table if it is big enough
+			if tw.tw.BytesLen() >= kMaxTableSize {
+				err = finish()
+				if err != nil {
+					return
+				}
+				snapSched = true
+				tw = nil
+			}
+		}
+
+		err = iter.Error()
+		if err != nil {
+			return
+		}
+
+		// Finish last table
+		if tw != nil && !tw.empty() {
+			err = finish()
+			if err != nil {
+				return
+			}
+			tw = nil
+		}
+		return
+	}, func() error {
+		for _, r := range rec.addedTables {
+			s.logf("table@build rollback @%d", r.num)
+			f := s.getTableFile(r.num)
+			if err := f.Remove(); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// Commit changes
+	d.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
+		stats[1].startTimer()
+		defer stats[1].stopTimer()
+		return s.commit(rec)
+	}, nil)
+
+	resultSize := int(int(stats[1].write))
+	s.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
+
+	// Save compaction stats
+	for i := range stats {
+		d.compStats[c.level+1].add(&stats[i])
+	}
+}
+
+func (d *DB) tableRangeCompaction(level int, min, max []byte) {
+	s := d.s
+	s.logf("table@compaction range L%d %q:%q", level, min, max)
+
+	if level >= 0 {
+		if c := s.getCompactionRange(level, min, max); c != nil {
+			d.tableCompaction(c, true)
+		}
+	} else {
+		v := s.version_NB()
+		m := 1
+		for i, t := range v.tables[1:] {
+			if t.isOverlaps(min, max, true, s.cmp) {
+				m = i + 1
+			}
+		}
+		for level := 0; level < m; level++ {
+			if c := s.getCompactionRange(level, min, max); c != nil {
+				d.tableCompaction(c, true)
+			}
+		}
+	}
+}
+
+func (d *DB) tableAutoCompaction() {
+	if c := d.s.pickCompaction(); c != nil {
+		d.tableCompaction(c, false)
+	}
+}
+
+func (d *DB) tableNeedCompaction() bool {
+	return d.s.version_NB().needCompaction()
+}
+
+func (d *DB) pauseCompaction(ch chan<- struct{}) {
+	select {
+	case ch <- struct{}{}:
+	case _, _ = <-d.closeC:
+		d.compactionExitTransact()
+	}
+}
+
+type cCmd interface {
+	ack(err error)
+}
+
+type cIdle struct {
+	ackC chan<- error
+}
+
+func (r cIdle) ack(err error) {
+	r.ackC <- err
+}
+
+type cRange struct {
+	level    int
+	min, max []byte
+	ackC     chan<- error
+}
+
+func (r cRange) ack(err error) {
+	defer func() {
+		recover()
+	}()
+	if r.ackC != nil {
+		r.ackC <- err
+	}
+}
+
+func (d *DB) compSendIdle(compC chan<- cCmd) error {
+	ch := make(chan error)
+	defer close(ch)
+	// Send cmd.
+	select {
+	case compC <- cIdle{ch}:
+	case err := <-d.compErrC:
+		return err
+	case _, _ = <-d.closeC:
+		return ErrClosed
+	}
+	// Wait cmd.
+	return <-ch
+}
+
+func (d *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
+	ch := make(chan error)
+	defer close(ch)
+	// Send cmd.
+	select {
+	case compC <- cRange{level, min, max, ch}:
+	case err := <-d.compErrC:
+		return err
+	case _, _ = <-d.closeC:
+		return ErrClosed
+	}
+	// Wait cmd.
+	select {
+	case err = <-d.compErrC:
+	case err = <-ch:
+	}
+	return err
+}
+
+func (d *DB) compTrigger(compTriggerC chan struct{}) {
+	select {
+	case compTriggerC <- struct{}{}:
+	default:
+	}
+}
+
+func (d *DB) mCompaction() {
+	var x cCmd
+
+	defer func() {
+		if x := recover(); x != nil {
+			if x != errCompactionTransactExiting {
+				panic(x)
+			}
+		}
+		if x != nil {
+			x.ack(ErrClosed)
+		}
+		d.closeW.Done()
+	}()
+
+	for {
+		select {
+		case _, _ = <-d.closeC:
+			return
+		case x = <-d.mcompCmdC:
+			d.memCompaction()
+			x.ack(nil)
+			x = nil
+		case <-d.mcompTriggerC:
+			d.memCompaction()
+		}
+	}
+}
+
+func (d *DB) tCompaction() {
+	var x cCmd
+	var ackQ []cCmd
+
+	defer func() {
+		if x := recover(); x != nil {
+			if x != errCompactionTransactExiting {
+				panic(x)
+			}
+		}
+		for i := range ackQ {
+			ackQ[i].ack(ErrClosed)
+			ackQ[i] = nil
+		}
+		if x != nil {
+			x.ack(ErrClosed)
+		}
+		d.closeW.Done()
+	}()
+
+	for {
+		if d.tableNeedCompaction() {
+			select {
+			case x = <-d.tcompCmdC:
+			case <-d.tcompTriggerC:
+			case _, _ = <-d.closeC:
+				return
+			case ch := <-d.tcompPauseC:
+				d.pauseCompaction(ch)
+				continue
+			default:
+			}
+		} else {
+			for i := range ackQ {
+				ackQ[i].ack(nil)
+				ackQ[i] = nil
+			}
+			ackQ = ackQ[:0]
+			select {
+			case x = <-d.tcompCmdC:
+			case <-d.tcompTriggerC:
+			case ch := <-d.tcompPauseC:
+				d.pauseCompaction(ch)
+				continue
+			case _, _ = <-d.closeC:
+				return
+			}
+		}
+		if x != nil {
+			switch cmd := x.(type) {
+			case cIdle:
+				ackQ = append(ackQ, x)
+			case cRange:
+				d.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
+				x.ack(nil)
+			}
+			x = nil
+		}
+		d.tableAutoCompaction()
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -0,0 +1,311 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+	"runtime"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
+)
+
+func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	s := db.s
+
+	em, fm := db.getMems()
+	v := s.version()
+
+	ti := v.getIterators(slice, ro)
+	n := len(ti) + 2
+	i := make([]iterator.Iterator, 0, n)
+	i = append(i, em.NewIterator(slice))
+	if fm != nil {
+		i = append(i, fm.NewIterator(slice))
+	}
+	i = append(i, ti...)
+	strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
+	mi := iterator.NewMergedIterator(i, s.cmp, strict)
+	mi.SetReleaser(&versionReleaser{v: v})
+	return mi
+}
+
+func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
+	var slice_ *util.Range
+	if slice != nil {
+		slice_ = &util.Range{}
+		if slice.Start != nil {
+			slice_.Start = newIKey(slice.Start, kMaxSeq, tSeek)
+		}
+		if slice.Limit != nil {
+			slice_.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
+		}
+	}
+	rawIter := db.newRawIterator(slice_, ro)
+	iter := &dbIter{
+		cmp:    db.s.cmp.cmp,
+		iter:   rawIter,
+		seq:    seq,
+		strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
+		key:    make([]byte, 0),
+		value:  make([]byte, 0),
+	}
+	runtime.SetFinalizer(iter, (*dbIter).Release)
+	return iter
+}
+
+type dir int
+
+const (
+	dirReleased dir = iota - 1
+	dirSOI
+	dirEOI
+	dirBackward
+	dirForward
+)
+
+// dbIter represent an interator states over a database session.
+type dbIter struct {
+	cmp    comparer.BasicComparer
+	iter   iterator.Iterator
+	seq    uint64
+	strict bool
+
+	dir      dir
+	key      []byte
+	value    []byte
+	err      error
+	releaser util.Releaser
+}
+
+func (i *dbIter) setErr(err error) {
+	i.err = err
+	i.key = nil
+	i.value = nil
+}
+
+func (i *dbIter) iterErr() {
+	if err := i.iter.Error(); err != nil {
+		i.setErr(err)
+	}
+}
+
+func (i *dbIter) Valid() bool {
+	return i.err == nil && i.dir > dirEOI
+}
+
+func (i *dbIter) First() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if i.iter.First() {
+		i.dir = dirSOI
+		return i.next()
+	}
+	i.dir = dirEOI
+	i.iterErr()
+	return false
+}
+
+func (i *dbIter) Last() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if i.iter.Last() {
+		return i.prev()
+	}
+	i.dir = dirSOI
+	i.iterErr()
+	return false
+}
+
+func (i *dbIter) Seek(key []byte) bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	ikey := newIKey(key, i.seq, tSeek)
+	if i.iter.Seek(ikey) {
+		i.dir = dirSOI
+		return i.next()
+	}
+	i.dir = dirEOI
+	i.iterErr()
+	return false
+}
+
+func (i *dbIter) next() bool {
+	for {
+		ukey, seq, t, ok := parseIkey(i.iter.Key())
+		if ok {
+			if seq <= i.seq {
+				switch t {
+				case tDel:
+					// Skip deleted key.
+					i.key = append(i.key[:0], ukey...)
+					i.dir = dirForward
+				case tVal:
+					if i.dir == dirSOI || i.cmp.Compare(ukey, i.key) > 0 {
+						i.key = append(i.key[:0], ukey...)
+						i.value = append(i.value[:0], i.iter.Value()...)
+						i.dir = dirForward
+						return true
+					}
+				}
+			}
+		} else if i.strict {
+			i.setErr(errInvalidIkey)
+			break
+		}
+		if !i.iter.Next() {
+			i.dir = dirEOI
+			i.iterErr()
+			break
+		}
+	}
+	return false
+}
+
+func (i *dbIter) Next() bool {
+	if i.dir == dirEOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) {
+		i.dir = dirEOI
+		i.iterErr()
+		return false
+	}
+	return i.next()
+}
+
+func (i *dbIter) prev() bool {
+	i.dir = dirBackward
+	del := true
+	if i.iter.Valid() {
+		for {
+			ukey, seq, t, ok := parseIkey(i.iter.Key())
+			if ok {
+				if seq <= i.seq {
+					if !del && i.cmp.Compare(ukey, i.key) < 0 {
+						return true
+					}
+					del = (t == tDel)
+					if !del {
+						i.key = append(i.key[:0], ukey...)
+						i.value = append(i.value[:0], i.iter.Value()...)
+					}
+				}
+			} else if i.strict {
+				i.setErr(errInvalidIkey)
+				return false
+			}
+			if !i.iter.Prev() {
+				break
+			}
+		}
+	}
+	if del {
+		i.dir = dirSOI
+		i.iterErr()
+		return false
+	}
+	return true
+}
+
+func (i *dbIter) Prev() bool {
+	if i.dir == dirSOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	switch i.dir {
+	case dirEOI:
+		return i.Last()
+	case dirForward:
+		for i.iter.Prev() {
+			ukey, _, _, ok := parseIkey(i.iter.Key())
+			if ok {
+				if i.cmp.Compare(ukey, i.key) < 0 {
+					goto cont
+				}
+			} else if i.strict {
+				i.setErr(errInvalidIkey)
+				return false
+			}
+		}
+		i.dir = dirSOI
+		i.iterErr()
+		return false
+	}
+
+cont:
+	return i.prev()
+}
+
+func (i *dbIter) Key() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.key
+}
+
+func (i *dbIter) Value() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.value
+}
+
+func (i *dbIter) Release() {
+	if i.dir != dirReleased {
+		// Clear the finalizer.
+		runtime.SetFinalizer(i, nil)
+
+		if i.releaser != nil {
+			i.releaser.Release()
+		}
+
+		i.dir = dirReleased
+		i.key = nil
+		i.value = nil
+		i.iter.Release()
+		i.iter = nil
+	}
+}
+
+func (i *dbIter) SetReleaser(releaser util.Releaser) {
+	if i.dir != dirReleased {
+		i.releaser = releaser
+	}
+}
+
+func (i *dbIter) Error() error {
+	return i.err
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -0,0 +1,165 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type snapshotElement struct {
+	seq uint64
+	ref int
+	// Next and previous pointers in the doubly-linked list of elements.
+	next, prev *snapshotElement
+}
+
+// Initialize the snapshot.
+func (db *DB) initSnapshot() {
+	db.snapsRoot.next = &db.snapsRoot
+	db.snapsRoot.prev = &db.snapsRoot
+}
+
+// Acquires a snapshot, based on latest sequence.
+func (db *DB) acquireSnapshot() *snapshotElement {
+	db.snapsMu.Lock()
+	seq := db.getSeq()
+	elem := db.snapsRoot.prev
+	if elem == &db.snapsRoot || elem.seq != seq {
+		at := db.snapsRoot.prev
+		next := at.next
+		elem = &snapshotElement{
+			seq:  seq,
+			prev: at,
+			next: next,
+		}
+		at.next = elem
+		next.prev = elem
+	}
+	elem.ref++
+	db.snapsMu.Unlock()
+	return elem
+}
+
+// Releases given snapshot element.
+func (db *DB) releaseSnapshot(elem *snapshotElement) {
+	if !db.isClosed() {
+		db.snapsMu.Lock()
+		elem.ref--
+		if elem.ref == 0 {
+			elem.prev.next = elem.next
+			elem.next.prev = elem.prev
+			elem.next = nil
+			elem.prev = nil
+		} else if elem.ref < 0 {
+			panic("leveldb: Snapshot: negative element reference")
+		}
+		db.snapsMu.Unlock()
+	}
+}
+
+// Gets minimum sequence that not being snapshoted.
+func (db *DB) minSeq() uint64 {
+	db.snapsMu.Lock()
+	defer db.snapsMu.Unlock()
+	elem := db.snapsRoot.prev
+	if elem != &db.snapsRoot {
+		return elem.seq
+	}
+	return db.getSeq()
+}
+
+// Snapshot is a DB snapshot.
+type Snapshot struct {
+	db       *DB
+	elem     *snapshotElement
+	mu       sync.Mutex
+	released bool
+}
+
+// Creates new snapshot object.
+func (db *DB) newSnapshot() *Snapshot {
+	p := &Snapshot{
+		db:   db,
+		elem: db.acquireSnapshot(),
+	}
+	runtime.SetFinalizer(p, (*Snapshot).Release)
+	return p
+}
+
+// Get gets the value for the given key. It returns ErrNotFound if
+// the DB does not contain the key.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Get returns.
+func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	db := p.db
+	err = db.ok()
+	if err != nil {
+		return
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.released {
+		err = ErrSnapshotReleased
+		return
+	}
+	return db.get(key, p.elem.seq, ro)
+}
+
+// NewIterator returns an iterator for the snapshot of the uderlying DB.
+// The returned iterator is not goroutine-safe, but it is safe to use
+// multiple iterators concurrently, with each in a dedicated goroutine.
+// It is also safe to use an iterator concurrently with modifying its
+// underlying DB. The resultant key/value pairs are guaranteed to be
+// consistent.
+//
+// Slice allows slicing the iterator to only contains keys in the given
+// range. A nil Range.Start is treated as a key before all keys in the
+// DB. And a nil Range.Limit is treated as a key after all keys in
+// the DB.
+//
+// The iterator must be released after use, by calling Release method.
+// Releasing the snapshot doesn't mean releasing the iterator too, the
+// iterator would be still valid until released.
+//
+// Also read Iterator documentation of the leveldb/iterator package.
+func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	db := p.db
+	if err := db.ok(); err != nil {
+		return iterator.NewEmptyIterator(err)
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.released {
+		return iterator.NewEmptyIterator(ErrSnapshotReleased)
+	}
+	return db.newIterator(p.elem.seq, slice, ro)
+}
+
+// Release releases the snapshot. This will not release any returned
+// iterators, the iterators would still be valid until released or the
+// underlying DB is closed.
+//
+// Other methods should not be called after the snapshot has been released.
+func (p *Snapshot) Release() {
+	p.mu.Lock()
+	if !p.released {
+		// Clear the finalizer.
+		runtime.SetFinalizer(p, nil)
+
+		p.released = true
+		p.db.releaseSnapshot(p.elem)
+		p.db = nil
+		p.elem = nil
+	}
+	p.mu.Unlock()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go

@@ -0,0 +1,114 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/journal"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+)
+
+// Get latest sequence number.
+func (d *DB) getSeq() uint64 {
+	return atomic.LoadUint64(&d.seq)
+}
+
+// Atomically adds delta to seq.
+func (d *DB) addSeq(delta uint64) {
+	atomic.AddUint64(&d.seq, delta)
+}
+
+// Create new memdb and froze the old one; need external synchronization.
+// newMem only called synchronously by the writer.
+func (d *DB) newMem(n int) (mem *memdb.DB, err error) {
+	s := d.s
+
+	num := s.allocFileNum()
+	file := s.getJournalFile(num)
+	w, err := file.Create()
+	if err != nil {
+		s.reuseFileNum(num)
+		return
+	}
+	d.memMu.Lock()
+	if d.journal == nil {
+		d.journal = journal.NewWriter(w)
+	} else {
+		d.journal.Reset(w)
+		d.journalWriter.Close()
+		d.frozenJournalFile = d.journalFile
+	}
+	d.journalWriter = w
+	d.journalFile = file
+	d.frozenMem = d.mem
+	d.mem = memdb.New(s.cmp, maxInt(d.s.o.GetWriteBuffer(), n))
+	mem = d.mem
+	// The seq only incremented by the writer.
+	d.frozenSeq = d.seq
+	d.memMu.Unlock()
+	return
+}
+
+// Get all memdbs.
+func (d *DB) getMems() (e *memdb.DB, f *memdb.DB) {
+	d.memMu.RLock()
+	defer d.memMu.RUnlock()
+	return d.mem, d.frozenMem
+}
+
+// Get frozen memdb.
+func (d *DB) getEffectiveMem() *memdb.DB {
+	d.memMu.RLock()
+	defer d.memMu.RUnlock()
+	return d.mem
+}
+
+// Check whether we has frozen memdb.
+func (d *DB) hasFrozenMem() bool {
+	d.memMu.RLock()
+	defer d.memMu.RUnlock()
+	return d.frozenMem != nil
+}
+
+// Get frozen memdb.
+func (d *DB) getFrozenMem() *memdb.DB {
+	d.memMu.RLock()
+	defer d.memMu.RUnlock()
+	return d.frozenMem
+}
+
+// Drop frozen memdb; assume that frozen memdb isn't nil.
+func (d *DB) dropFrozenMem() {
+	d.memMu.Lock()
+	if err := d.frozenJournalFile.Remove(); err != nil {
+		d.s.logf("journal@remove removing @%d %q", d.frozenJournalFile.Num(), err)
+	} else {
+		d.s.logf("journal@remove removed @%d", d.frozenJournalFile.Num())
+	}
+	d.frozenJournalFile = nil
+	d.frozenMem = nil
+	d.memMu.Unlock()
+}
+
+// Set closed flag; return true if not already closed.
+func (d *DB) setClosed() bool {
+	return atomic.CompareAndSwapUint32(&d.closed, 0, 1)
+}
+
+// Check whether DB was closed.
+func (d *DB) isClosed() bool {
+	return atomic.LoadUint32(&d.closed) != 0
+}
+
+// Check read ok status.
+func (d *DB) ok() error {
+	if d.isClosed() {
+		return ErrClosed
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// Reader is the interface that wraps basic Get and NewIterator methods.
+// This interface implemented by both DB and Snapshot.
+type Reader interface {
+	Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
+	NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
+}
+
+type Sizes []uint64
+
+// Sum returns sum of the sizes.
+func (p Sizes) Sum() (n uint64) {
+	for _, s := range p {
+		n += s
+	}
+	return n
+}
+
+// Check and clean files.
+func (d *DB) checkAndCleanFiles() error {
+	s := d.s
+
+	v := s.version_NB()
+	tables := make(map[uint64]bool)
+	for _, tt := range v.tables {
+		for _, t := range tt {
+			tables[t.file.Num()] = false
+		}
+	}
+
+	ff, err := s.getFiles(storage.TypeAll)
+	if err != nil {
+		return err
+	}
+
+	var nTables int
+	var rem []storage.File
+	for _, f := range ff {
+		keep := true
+		switch f.Type() {
+		case storage.TypeManifest:
+			keep = f.Num() >= s.manifestFile.Num()
+		case storage.TypeJournal:
+			if d.frozenJournalFile != nil {
+				keep = f.Num() >= d.frozenJournalFile.Num()
+			} else {
+				keep = f.Num() >= d.journalFile.Num()
+			}
+		case storage.TypeTable:
+			_, keep = tables[f.Num()]
+			if keep {
+				tables[f.Num()] = true
+				nTables++
+			}
+		}
+
+		if !keep {
+			rem = append(rem, f)
+		}
+	}
+
+	if nTables != len(tables) {
+		for num, present := range tables {
+			if !present {
+				s.logf("db@janitor table missing @%d", num)
+			}
+		}
+		return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")}
+	}
+
+	s.logf("db@janitor F·%d G·%d", len(ff), len(rem))
+	for _, f := range rem {
+		s.logf("db@janitor removing %s-%d", f.Type(), f.Num())
+		if err := f.Remove(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go

@@ -0,0 +1,280 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func (d *DB) writeJournal(b *Batch) error {
+	w, err := d.journal.Next()
+	if err != nil {
+		return err
+	}
+	if _, err := w.Write(b.encode()); err != nil {
+		return err
+	}
+	if err := d.journal.Flush(); err != nil {
+		return err
+	}
+	if b.sync {
+		return d.journalWriter.Sync()
+	}
+	return nil
+}
+
+func (d *DB) jWriter() {
+	defer d.closeW.Done()
+	for {
+		select {
+		case b := <-d.journalC:
+			if b != nil {
+				d.journalAckC <- d.writeJournal(b)
+			}
+		case _, _ = <-d.closeC:
+			return
+		}
+	}
+}
+
+func (d *DB) rotateMem(n int) (mem *memdb.DB, err error) {
+	// Wait for pending memdb compaction.
+	err = d.compSendIdle(d.mcompCmdC)
+	if err != nil {
+		return
+	}
+
+	// Create new memdb and journal.
+	mem, err = d.newMem(n)
+	if err != nil {
+		return
+	}
+
+	// Schedule memdb compaction.
+	d.compTrigger(d.mcompTriggerC)
+	return
+}
+
+func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
+	s := d.s
+
+	delayed := false
+	flush := func() bool {
+		v := s.version()
+		defer v.release()
+		mem = d.getEffectiveMem()
+		nn = mem.Free()
+		switch {
+		case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
+			delayed = true
+			time.Sleep(time.Millisecond)
+		case nn >= n:
+			return false
+		case v.tLen(0) >= kL0_StopWritesTrigger:
+			delayed = true
+			err = d.compSendIdle(d.tcompCmdC)
+			if err != nil {
+				return false
+			}
+		default:
+			// Allow memdb to grow if it has no entry.
+			if mem.Len() == 0 {
+				nn = n
+				return false
+			}
+			mem, err = d.rotateMem(n)
+			nn = mem.Free()
+			return false
+		}
+		return true
+	}
+	start := time.Now()
+	for flush() {
+	}
+	if delayed {
+		s.logf("db@write delayed T·%v", time.Since(start))
+	}
+	return
+}
+
+// Write apply the given batch to the DB. The batch will be applied
+// sequentially.
+//
+// It is safe to modify the contents of the arguments after Write returns.
+func (d *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
+	err = d.ok()
+	if err != nil || b == nil || b.len() == 0 {
+		return
+	}
+
+	b.init(wo.GetSync())
+
+	// The write happen synchronously.
+retry:
+	select {
+	case d.writeC <- b:
+		if <-d.writeMergedC {
+			return <-d.writeAckC
+		}
+		goto retry
+	case d.writeLockC <- struct{}{}:
+	case _, _ = <-d.closeC:
+		return ErrClosed
+	}
+
+	merged := 0
+	defer func() {
+		<-d.writeLockC
+		for i := 0; i < merged; i++ {
+			d.writeAckC <- err
+		}
+	}()
+
+	mem, memFree, err := d.flush(b.size())
+	if err != nil {
+		return
+	}
+
+	// Calculate maximum size of the batch.
+	m := 1 << 20
+	if x := b.size(); x <= 128<<10 {
+		m = x + (128 << 10)
+	}
+	m = minInt(m, memFree)
+
+	// Merge with other batch.
+drain:
+	for b.size() < m && !b.sync {
+		select {
+		case nb := <-d.writeC:
+			if b.size()+nb.size() <= m {
+				b.append(nb)
+				d.writeMergedC <- true
+				merged++
+			} else {
+				d.writeMergedC <- false
+				break drain
+			}
+		default:
+			break drain
+		}
+	}
+
+	// Set batch first seq number relative from last seq.
+	b.seq = d.seq + 1
+
+	// Write journal concurrently if it is large enough.
+	if b.size() >= (128 << 10) {
+		// Push the write batch to the journal writer
+		select {
+		case _, _ = <-d.closeC:
+			err = ErrClosed
+			return
+		case d.journalC <- b:
+			// Write into memdb
+			b.memReplay(mem)
+		}
+		// Wait for journal writer
+		select {
+		case _, _ = <-d.closeC:
+			err = ErrClosed
+			return
+		case err = <-d.journalAckC:
+			if err != nil {
+				// Revert memdb if error detected
+				b.revertMemReplay(mem)
+				return
+			}
+		}
+	} else {
+		err = d.writeJournal(b)
+		if err != nil {
+			return
+		}
+		b.memReplay(mem)
+	}
+
+	// Set last seq number.
+	d.addSeq(uint64(b.len()))
+
+	if b.size() >= memFree {
+		d.rotateMem(0)
+	}
+	return
+}
+
+// Put sets the value for the given key. It overwrites any previous value
+// for that key; a DB is not a multi-map.
+//
+// It is safe to modify the contents of the arguments after Put returns.
+func (d *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
+	b := new(Batch)
+	b.Put(key, value)
+	return d.Write(b, wo)
+}
+
+// Delete deletes the value for the given key. It returns ErrNotFound if
+// the DB does not contain the key.
+//
+// It is safe to modify the contents of the arguments after Delete returns.
+func (d *DB) Delete(key []byte, wo *opt.WriteOptions) error {
+	b := new(Batch)
+	b.Delete(key)
+	return d.Write(b, wo)
+}
+
+func isMemOverlaps(ucmp comparer.BasicComparer, mem *memdb.DB, min, max []byte) bool {
+	iter := mem.NewIterator(nil)
+	defer iter.Release()
+	return (max == nil || (iter.First() && ucmp.Compare(max, iKey(iter.Key()).ukey()) >= 0)) &&
+		(min == nil || (iter.Last() && ucmp.Compare(min, iKey(iter.Key()).ukey()) <= 0))
+}
+
+// CompactRange compacts the underlying DB for the given key range.
+// In particular, deleted and overwritten versions are discarded,
+// and the data is rearranged to reduce the cost of operations
+// needed to access the data. This operation should typically only
+// be invoked by users who understand the underlying implementation.
+//
+// A nil Range.Start is treated as a key before all keys in the DB.
+// And a nil Range.Limit is treated as a key after all keys in the DB.
+// Therefore if both is nil then it will compact entire DB.
+func (d *DB) CompactRange(r util.Range) error {
+	if err := d.ok(); err != nil {
+		return err
+	}
+
+	select {
+	case d.writeLockC <- struct{}{}:
+	case _, _ = <-d.closeC:
+		return ErrClosed
+	}
+
+	// Check for overlaps in memdb.
+	mem := d.getEffectiveMem()
+	if isMemOverlaps(d.s.cmp.cmp, mem, r.Start, r.Limit) {
+		// Memdb compaction.
+		if _, err := d.rotateMem(0); err != nil {
+			<-d.writeLockC
+			return err
+		}
+		<-d.writeLockC
+		if err := d.compSendIdle(d.mcompCmdC); err != nil {
+			return err
+		}
+	} else {
+		<-d.writeLockC
+	}
+
+	// Table compaction.
+	return d.compSendRange(d.tcompCmdC, -1, r.Start, r.Limit)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package leveldb provides implementation of LevelDB key/value database.
+//
+// Create or open a database:
+//
+//	db, err := leveldb.OpenFile("path/to/db", nil)
+//	...
+//	defer db.Close()
+//	...
+//
+// Read or modify the database content:
+//
+//	// Remember that the contents of the returned slice should not be modified.
+//	data, err := db.Get([]byte("key"), nil)
+//	...
+//	err = db.Put([]byte("key"), []byte("value"), nil)
+//	...
+//	err = db.Delete([]byte("key"), nil)
+//	...
+//
+// Iterate over database content:
+//
+//	iter := db.NewIterator(nil, nil)
+//	for iter.Next() {
+//		// Remember that the contents of the returned slice should not be modified, and
+//		// only valid until the next call to Next.
+//		key := iter.Key()
+//		value := iter.Value()
+//		...
+//	}
+//	iter.Release()
+//	err = iter.Error()
+//	...
+//
+// Seek-then-Iterate:
+//
+// 	iter := db.NewIterator(nil, nil)
+// 	for ok := iter.Seek(key); ok; ok = iter.Next() {
+// 		// Use key/value.
+// 		...
+// 	}
+// 	iter.Release()
+// 	err = iter.Error()
+// 	...
+//
+// Iterate over subset of database content:
+//
+// 	iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil)
+// 	for iter.Next() {
+// 		// Use key/value.
+// 		...
+// 	}
+// 	iter.Release()
+// 	err = iter.Error()
+// 	...
+//
+// Batch writes:
+//
+//	batch := new(leveldb.Batch)
+//	batch.Put([]byte("foo"), []byte("value"))
+//	batch.Put([]byte("bar"), []byte("another value"))
+//	batch.Delete([]byte("baz"))
+//	err = db.Write(batch, nil)
+//	...
+//
+// Use bloom filter:
+//
+//	o := &opt.Options{
+//		Filter: filter.NewBloomFilter(10),
+//	}
+//	db, err := leveldb.OpenFile("path/to/db", o)
+//	...
+//	defer db.Close()
+//	...
+package leveldb

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/error.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	ErrNotFound         = util.ErrNotFound
+	ErrSnapshotReleased = errors.New("leveldb: snapshot released")
+	ErrIterReleased     = errors.New("leveldb: iterator released")
+	ErrClosed           = errors.New("leveldb: closed")
+)
+
+type CorruptionType int
+
+const (
+	CorruptedManifest CorruptionType = iota
+	MissingFiles
+)
+
+// ErrCorrupted is the type that wraps errors that indicate corruption in
+// the database.
+type ErrCorrupted struct {
+	Type CorruptionType
+	Err  error
+}
+
+func (e ErrCorrupted) Error() string {
+	return e.Err.Error()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+var _ = testutil.Defer(func() {
+	Describe("Leveldb external", func() {
+		o := &opt.Options{
+			BlockCache:           opt.NoCache,
+			BlockRestartInterval: 5,
+			BlockSize:            50,
+			Compression:          opt.NoCompression,
+			MaxOpenFiles:         0,
+			Strict:               opt.StrictAll,
+			WriteBuffer:          1000,
+		}
+
+		Describe("write test", func() {
+			It("should do write correctly", func(done Done) {
+				db := newTestingDB(o, nil, nil)
+				t := testutil.DBTesting{
+					DB:      db,
+					Deleted: testutil.KeyValue_Generate(nil, 500, 1, 50, 5, 5).Clone(),
+				}
+				testutil.DoDBTesting(&t)
+				db.TestClose()
+				done <- true
+			}, 9.0)
+		})
+
+		Describe("read test", func() {
+			testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
+				// Building the DB.
+				db := newTestingDB(o, nil, nil)
+				kv.IterateShuffled(nil, func(i int, key, value []byte) {
+					err := db.TestPut(key, value)
+					Expect(err).NotTo(HaveOccurred())
+				})
+				testutil.Defer("teardown", func() {
+					db.TestClose()
+				})
+
+				return db
+			})
+		})
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/filter"
+)
+
+type iFilter struct {
+	filter.Filter
+}
+
+func (f iFilter) Contains(filter, key []byte) bool {
+	return f.Filter.Contains(filter, iKey(key).ukey())
+}
+
+func (f iFilter) NewGenerator() filter.FilterGenerator {
+	return iFilterGenerator{f.Filter.NewGenerator()}
+}
+
+type iFilterGenerator struct {
+	filter.FilterGenerator
+}
+
+func (g iFilterGenerator) Add(key []byte) {
+	g.FilterGenerator.Add(iKey(key).ukey())
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/bloom.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package filter
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func bloomHash(key []byte) uint32 {
+	return util.Hash(key, 0xbc9f1d34)
+}
+
+type bloomFilter int
+
+// The bloom filter serializes its parameters and is backward compatible
+// with respect to them. Therefor, its parameters are not added to its
+// name.
+func (bloomFilter) Name() string {
+	return "leveldb.BuiltinBloomFilter"
+}
+
+func (f bloomFilter) Contains(filter, key []byte) bool {
+	nBytes := len(filter) - 1
+	if nBytes < 1 {
+		return false
+	}
+	nBits := uint32(nBytes * 8)
+
+	// Use the encoded k so that we can read filters generated by
+	// bloom filters created using different parameters.
+	k := filter[nBytes]
+	if k > 30 {
+		// Reserved for potentially new encodings for short bloom filters.
+		// Consider it a match.
+		return true
+	}
+
+	kh := bloomHash(key)
+	delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
+	for j := uint8(0); j < k; j++ {
+		bitpos := kh % nBits
+		if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 {
+			return false
+		}
+		kh += delta
+	}
+	return true
+}
+
+func (f bloomFilter) NewGenerator() FilterGenerator {
+	// Round down to reduce probing cost a little bit.
+	k := uint8(f * 69 / 100) // 0.69 =~ ln(2)
+	if k < 1 {
+		k = 1
+	} else if k > 30 {
+		k = 30
+	}
+	return &bloomFilterGenerator{
+		n: int(f),
+		k: k,
+	}
+}
+
+type bloomFilterGenerator struct {
+	n int
+	k uint8
+
+	keyHashes []uint32
+}
+
+func (g *bloomFilterGenerator) Add(key []byte) {
+	// Use double-hashing to generate a sequence of hash values.
+	// See analysis in [Kirsch,Mitzenmacher 2006].
+	g.keyHashes = append(g.keyHashes, bloomHash(key))
+}
+
+func (g *bloomFilterGenerator) Generate(b Buffer) {
+	// Compute bloom filter size (in both bits and bytes)
+	nBits := uint32(len(g.keyHashes) * g.n)
+	// For small n, we can see a very high false positive rate.  Fix it
+	// by enforcing a minimum bloom filter length.
+	if nBits < 64 {
+		nBits = 64
+	}
+	nBytes := (nBits + 7) / 8
+	nBits = nBytes * 8
+
+	dest := b.Alloc(int(nBytes) + 1)
+	dest[nBytes] = g.k
+	for _, kh := range g.keyHashes {
+		delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
+		for j := uint8(0); j < g.k; j++ {
+			bitpos := kh % nBits
+			dest[bitpos/8] |= (1 << (bitpos % 8))
+			kh += delta
+		}
+	}
+
+	g.keyHashes = g.keyHashes[:0]
+}
+
+// NewBloomFilter creates a new initialized bloom filter for given
+// bitsPerKey.
+//
+// Since bitsPerKey is persisted individually for each bloom filter
+// serialization, bloom filters are backwards compatible with respect to
+// changing bitsPerKey. This means that no big performance penalty will
+// be experienced when changing the parameter. See documentation for
+// opt.Options.Filter for more information.
+func NewBloomFilter(bitsPerKey int) Filter {
+	return bloomFilter(bitsPerKey)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package filter
+
+import (
+	"encoding/binary"
+	"github.com/syndtr/goleveldb/leveldb/util"
+	"testing"
+)
+
+type harness struct {
+	t *testing.T
+
+	bloom     Filter
+	generator FilterGenerator
+	filter    []byte
+}
+
+func newHarness(t *testing.T) *harness {
+	bloom := NewBloomFilter(10)
+	return &harness{
+		t:         t,
+		bloom:     bloom,
+		generator: bloom.NewGenerator(),
+	}
+}
+
+func (h *harness) add(key []byte) {
+	h.generator.Add(key)
+}
+
+func (h *harness) addNum(key uint32) {
+	var b [4]byte
+	binary.LittleEndian.PutUint32(b[:], key)
+	h.add(b[:])
+}
+
+func (h *harness) build() {
+	b := &util.Buffer{}
+	h.generator.Generate(b)
+	h.filter = b.Bytes()
+}
+
+func (h *harness) reset() {
+	h.filter = nil
+}
+
+func (h *harness) filterLen() int {
+	return len(h.filter)
+}
+
+func (h *harness) assert(key []byte, want, silent bool) bool {
+	got := h.bloom.Contains(h.filter, key)
+	if !silent && got != want {
+		h.t.Errorf("assert on '%v' failed got '%v', want '%v'", key, got, want)
+	}
+	return got
+}
+
+func (h *harness) assertNum(key uint32, want, silent bool) bool {
+	var b [4]byte
+	binary.LittleEndian.PutUint32(b[:], key)
+	return h.assert(b[:], want, silent)
+}
+
+func TestBloomFilter_Empty(t *testing.T) {
+	h := newHarness(t)
+	h.build()
+	h.assert([]byte("hello"), false, false)
+	h.assert([]byte("world"), false, false)
+}
+
+func TestBloomFilter_Small(t *testing.T) {
+	h := newHarness(t)
+	h.add([]byte("hello"))
+	h.add([]byte("world"))
+	h.build()
+	h.assert([]byte("hello"), true, false)
+	h.assert([]byte("world"), true, false)
+	h.assert([]byte("x"), false, false)
+	h.assert([]byte("foo"), false, false)
+}
+
+func nextN(n int) int {
+	switch {
+	case n < 10:
+		n += 1
+	case n < 100:
+		n += 10
+	case n < 1000:
+		n += 100
+	default:
+		n += 1000
+	}
+	return n
+}
+
+func TestBloomFilter_VaryingLengths(t *testing.T) {
+	h := newHarness(t)
+	var mediocre, good int
+	for n := 1; n < 10000; n = nextN(n) {
+		h.reset()
+		for i := 0; i < n; i++ {
+			h.addNum(uint32(i))
+		}
+		h.build()
+
+		got := h.filterLen()
+		want := (n * 10 / 8) + 40
+		if got > want {
+			t.Errorf("filter len test failed, '%d' > '%d'", got, want)
+		}
+
+		for i := 0; i < n; i++ {
+			h.assertNum(uint32(i), true, false)
+		}
+
+		var rate float32
+		for i := 0; i < 10000; i++ {
+			if h.assertNum(uint32(i+1000000000), true, true) {
+				rate++
+			}
+		}
+		rate /= 10000
+		if rate > 0.02 {
+			t.Errorf("false positive rate is more than 2%%, got %v, at len %d", rate, n)
+		}
+		if rate > 0.0125 {
+			mediocre++
+		} else {
+			good++
+		}
+	}
+	t.Logf("false positive rate: %d good, %d mediocre", good, mediocre)
+	if mediocre > good/5 {
+		t.Error("mediocre false positive rate is more than expected")
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/filter.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package filter provides interface and implementation of probabilistic
+// data structure.
+//
+// The filter is resposible for creating small filter from a set of keys.
+// These filter will then used to test whether a key is a member of the set.
+// In many cases, a filter can cut down the number of disk seeks from a
+// handful to a single disk seek per DB.Get call.
+package filter
+
+// Buffer is the interface that wraps basic Alloc, Write and WriteByte methods.
+type Buffer interface {
+	// Alloc allocs n bytes of slice from the buffer. This also advancing
+	// write offset.
+	Alloc(n int) []byte
+
+	// Write appends the contents of p to the buffer.
+	Write(p []byte) (n int, err error)
+
+	// WriteByte appends the byte c to the buffer.
+	WriteByte(c byte) error
+}
+
+// Filter is the filter.
+type Filter interface {
+	// Name returns the name of this policy.
+	//
+	// Note that if the filter encoding changes in an incompatible way,
+	// the name returned by this method must be changed. Otherwise, old
+	// incompatible filters may be passed to methods of this type.
+	Name() string
+
+	// NewGenerator creates a new filter generator.
+	NewGenerator() FilterGenerator
+
+	// Contains returns true if the filter contains the given key.
+	//
+	// The filter are filters generated by the filter generator.
+	Contains(filter, key []byte) bool
+}
+
+// FilterGenerator is the filter generator.
+type FilterGenerator interface {
+	// Add adds a key to the filter generator.
+	//
+	// The key may become invalid after call to this method end, therefor
+	// key must be copied if implementation require keeping key for later
+	// use. The key should not modified directly, doing so may cause
+	// undefined results.
+	Add(key []byte)
+
+	// Generate generates filters based on keys passed so far. After call
+	// to Generate the filter generator maybe resetted, depends on implementation.
+	Generate(b Buffer)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// BasicArray is the interface that wraps basic Len and Search method.
+type BasicArray interface {
+	// Len returns length of the array.
+	Len() int
+
+	// Search finds smallest index that point to a key that is greater
+	// than or equal to the given key.
+	Search(key []byte) int
+}
+
+// Array is the interface that wraps BasicArray and basic Index method.
+type Array interface {
+	BasicArray
+
+	// Index returns key/value pair with index of i.
+	Index(i int) (key, value []byte)
+}
+
+// Array is the interface that wraps BasicArray and basic Get method.
+type ArrayIndexer interface {
+	BasicArray
+
+	// Get returns a new data iterator with index of i.
+	Get(i int) Iterator
+}
+
+type basicArrayIterator struct {
+	util.BasicReleaser
+	array BasicArray
+	pos   int
+}
+
+func (i *basicArrayIterator) Valid() bool {
+	return i.pos >= 0 && i.pos < i.array.Len()
+}
+
+func (i *basicArrayIterator) First() bool {
+	if i.array.Len() == 0 {
+		i.pos = -1
+		return false
+	}
+	i.pos = 0
+	return true
+}
+
+func (i *basicArrayIterator) Last() bool {
+	n := i.array.Len()
+	if n == 0 {
+		i.pos = 0
+		return false
+	}
+	i.pos = n - 1
+	return true
+}
+
+func (i *basicArrayIterator) Seek(key []byte) bool {
+	n := i.array.Len()
+	if n == 0 {
+		i.pos = 0
+		return false
+	}
+	i.pos = i.array.Search(key)
+	if i.pos >= n {
+		return false
+	}
+	return true
+}
+
+func (i *basicArrayIterator) Next() bool {
+	i.pos++
+	if n := i.array.Len(); i.pos >= n {
+		i.pos = n
+		return false
+	}
+	return true
+}
+
+func (i *basicArrayIterator) Prev() bool {
+	i.pos--
+	if i.pos < 0 {
+		i.pos = -1
+		return false
+	}
+	return true
+}
+
+func (i *basicArrayIterator) Error() error { return nil }
+
+type arrayIterator struct {
+	basicArrayIterator
+	array      Array
+	pos        int
+	key, value []byte
+}
+
+func (i *arrayIterator) updateKV() {
+	if i.pos == i.basicArrayIterator.pos {
+		return
+	}
+	i.pos = i.basicArrayIterator.pos
+	if i.Valid() {
+		i.key, i.value = i.array.Index(i.pos)
+	} else {
+		i.key = nil
+		i.value = nil
+	}
+}
+
+func (i *arrayIterator) Key() []byte {
+	i.updateKV()
+	return i.key
+}
+
+func (i *arrayIterator) Value() []byte {
+	i.updateKV()
+	return i.value
+}
+
+type arrayIteratorIndexer struct {
+	basicArrayIterator
+	array ArrayIndexer
+}
+
+func (i *arrayIteratorIndexer) Get() Iterator {
+	if i.Valid() {
+		return i.array.Get(i.basicArrayIterator.pos)
+	}
+	return nil
+}
+
+// NewArrayIterator returns an iterator from the given array.
+func NewArrayIterator(array Array) Iterator {
+	return &arrayIterator{
+		basicArrayIterator: basicArrayIterator{array: array, pos: -1},
+		array:              array,
+		pos:                -1,
+	}
+}
+
+// NewArrayIndexer returns an index iterator from the given array.
+func NewArrayIndexer(array ArrayIndexer) IteratorIndexer {
+	return &arrayIteratorIndexer{
+		basicArrayIterator: basicArrayIterator{array: array, pos: -1},
+		array:              array,
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator_test
+
+import (
+	. "github.com/onsi/ginkgo"
+
+	. "github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+var _ = testutil.Defer(func() {
+	Describe("Array iterator", func() {
+		It("Should iterates and seeks correctly", func() {
+			// Build key/value.
+			kv := testutil.KeyValue_Generate(nil, 70, 1, 5, 3, 3)
+
+			// Test the iterator.
+			t := testutil.IteratorTesting{
+				KeyValue: kv.Clone(),
+				Iter:     NewArrayIterator(kv),
+			}
+			testutil.DoIteratorTesting(&t)
+		})
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go

@@ -0,0 +1,221 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// IteratorIndexer is the interface that wraps CommonIterator and basic Get
+// method. IteratorIndexer provides index for indexed iterator.
+type IteratorIndexer interface {
+	CommonIterator
+
+	// Get returns a new data iterator for the current position, or nil if
+	// done.
+	Get() Iterator
+}
+
+type indexedIterator struct {
+	util.BasicReleaser
+	index     IteratorIndexer
+	strict    bool
+	strictGet bool
+
+	data Iterator
+	err  error
+	errf func(err error)
+}
+
+func (i *indexedIterator) setData() {
+	if i.data != nil {
+		i.data.Release()
+	}
+	i.data = i.index.Get()
+	if i.strictGet {
+		if err := i.data.Error(); err != nil {
+			i.err = err
+		}
+	}
+}
+
+func (i *indexedIterator) clearData() {
+	if i.data != nil {
+		i.data.Release()
+	}
+	i.data = nil
+}
+
+func (i *indexedIterator) dataErr() bool {
+	if i.errf != nil {
+		if err := i.data.Error(); err != nil {
+			i.errf(err)
+		}
+	}
+	if i.strict {
+		if err := i.data.Error(); err != nil {
+			i.err = err
+			return true
+		}
+	}
+	return false
+}
+
+func (i *indexedIterator) Valid() bool {
+	return i.data != nil && i.data.Valid()
+}
+
+func (i *indexedIterator) First() bool {
+	if i.err != nil {
+		return false
+	}
+
+	if !i.index.First() {
+		i.clearData()
+		return false
+	}
+	i.setData()
+	return i.Next()
+}
+
+func (i *indexedIterator) Last() bool {
+	if i.err != nil {
+		return false
+	}
+
+	if !i.index.Last() {
+		i.clearData()
+		return false
+	}
+	i.setData()
+	if !i.data.Last() {
+		if i.dataErr() {
+			return false
+		}
+		i.clearData()
+		return i.Prev()
+	}
+	return true
+}
+
+func (i *indexedIterator) Seek(key []byte) bool {
+	if i.err != nil {
+		return false
+	}
+
+	if !i.index.Seek(key) {
+		i.clearData()
+		return false
+	}
+	i.setData()
+	if !i.data.Seek(key) {
+		if i.dataErr() {
+			return false
+		}
+		i.clearData()
+		return i.Next()
+	}
+	return true
+}
+
+func (i *indexedIterator) Next() bool {
+	if i.err != nil {
+		return false
+	}
+
+	switch {
+	case i.data != nil && !i.data.Next():
+		if i.dataErr() {
+			return false
+		}
+		i.clearData()
+		fallthrough
+	case i.data == nil:
+		if !i.index.Next() {
+			return false
+		}
+		i.setData()
+		return i.Next()
+	}
+	return true
+}
+
+func (i *indexedIterator) Prev() bool {
+	if i.err != nil {
+		return false
+	}
+
+	switch {
+	case i.data != nil && !i.data.Prev():
+		if i.dataErr() {
+			return false
+		}
+		i.clearData()
+		fallthrough
+	case i.data == nil:
+		if !i.index.Prev() {
+			return false
+		}
+		i.setData()
+		if !i.data.Last() {
+			if i.dataErr() {
+				return false
+			}
+			i.clearData()
+			return i.Prev()
+		}
+	}
+	return true
+}
+
+func (i *indexedIterator) Key() []byte {
+	if i.data == nil {
+		return nil
+	}
+	return i.data.Key()
+}
+
+func (i *indexedIterator) Value() []byte {
+	if i.data == nil {
+		return nil
+	}
+	return i.data.Value()
+}
+
+func (i *indexedIterator) Release() {
+	i.clearData()
+	i.index.Release()
+	i.BasicReleaser.Release()
+}
+
+func (i *indexedIterator) Error() error {
+	if i.err != nil {
+		return i.err
+	}
+	if err := i.index.Error(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (i *indexedIterator) SetErrorCallback(f func(err error)) {
+	i.errf = f
+}
+
+// NewIndexedIterator returns an indexed iterator. An index is iterator
+// that returns another iterator, a data iterator. A data iterator is the
+// iterator that contains actual key/value pairs.
+//
+// If strict is true then error yield by data iterator will halt the indexed
+// iterator, on contrary if strict is false then the indexed iterator will
+// ignore those error and move on to the next index. If strictGet is true and
+// index.Get() yield an 'error iterator' then the indexed iterator will be halted.
+// An 'error iterator' is iterator which its Error() method always return non-nil
+// even before any 'seeks method' is called.
+func NewIndexedIterator(index IteratorIndexer, strict, strictGet bool) Iterator {
+	return &indexedIterator{index: index, strict: strict, strictGet: strictGet}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator_test
+
+import (
+	"sort"
+
+	. "github.com/onsi/ginkgo"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	. "github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+type keyValue struct {
+	key []byte
+	testutil.KeyValue
+}
+
+type keyValueIndex []keyValue
+
+func (x keyValueIndex) Search(key []byte) int {
+	return sort.Search(x.Len(), func(i int) bool {
+		return comparer.DefaultComparer.Compare(x[i].key, key) >= 0
+	})
+}
+
+func (x keyValueIndex) Len() int                        { return len(x) }
+func (x keyValueIndex) Index(i int) (key, value []byte) { return x[i].key, nil }
+func (x keyValueIndex) Get(i int) Iterator              { return NewArrayIterator(x[i]) }
+
+var _ = testutil.Defer(func() {
+	Describe("Indexed iterator", func() {
+		Test := func(n ...int) func() {
+			if len(n) == 0 {
+				rnd := testutil.NewRand()
+				n = make([]int, rnd.Intn(17)+3)
+				for i := range n {
+					n[i] = rnd.Intn(19) + 1
+				}
+			}
+
+			return func() {
+				It("Should iterates and seeks correctly", func(done Done) {
+					// Build key/value.
+					index := make(keyValueIndex, len(n))
+					sum := 0
+					for _, x := range n {
+						sum += x
+					}
+					kv := testutil.KeyValue_Generate(nil, sum, 1, 10, 4, 4)
+					for i, j := 0, 0; i < len(n); i++ {
+						for x := n[i]; x > 0; x-- {
+							key, value := kv.Index(j)
+							index[i].key = key
+							index[i].Put(key, value)
+							j++
+						}
+					}
+
+					// Test the iterator.
+					t := testutil.IteratorTesting{
+						KeyValue: kv.Clone(),
+						Iter:     NewIndexedIterator(NewArrayIndexer(index), true, true),
+					}
+					testutil.DoIteratorTesting(&t)
+					done <- true
+				}, 1.5)
+			}
+		}
+
+		Describe("with 100 keys", Test(100))
+		Describe("with 50-50 keys", Test(50, 50))
+		Describe("with 50-1 keys", Test(50, 1))
+		Describe("with 50-1-50 keys", Test(50, 1, 50))
+		Describe("with 1-50 keys", Test(1, 50))
+		Describe("with random N-keys", Test())
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package iterator provides interface and implementation to traverse over
+// contents of a database.
+package iterator
+
+import (
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// IteratorSeeker is the interface that wraps the 'seeks method'.
+type IteratorSeeker interface {
+	// First moves the iterator to the first key/value pair. If the iterator
+	// only contains one key/value pair then First and Last whould moves
+	// to the same key/value pair.
+	// It returns whether such pair exist.
+	First() bool
+
+	// Last moves the iterator to the last key/value pair. If the iterator
+	// only contains one key/value pair then First and Last whould moves
+	// to the same key/value pair.
+	// It returns whether such pair exist.
+	Last() bool
+
+	// Seek moves the iterator to the first key/value pair whose key is greater
+	// than or equal to the given key.
+	// It returns whether such pair exist.
+	//
+	// It is safe to modify the contents of the argument after Seek returns.
+	Seek(key []byte) bool
+
+	// Next moves the iterator to the next key/value pair.
+	// It returns whether the iterator is exhausted.
+	Next() bool
+
+	// Prev moves the iterator to the previous key/value pair.
+	// It returns whether the iterator is exhausted.
+	Prev() bool
+}
+
+// CommonIterator is the interface that wraps common interator methods.
+type CommonIterator interface {
+	IteratorSeeker
+
+	// util.Releaser is the interface that wraps basic Release method.
+	// When called Release will releases any resources associated with the
+	// iterator.
+	util.Releaser
+
+	// util.ReleaseSetter is the interface that wraps the basic SetReleaser
+	// method.
+	util.ReleaseSetter
+
+	// TODO: Remove this when ready.
+	Valid() bool
+
+	// Error returns any accumulated error. Exhausting all the key/value pairs
+	// is not considered to be an error.
+	Error() error
+}
+
+// Iterator iterates over a DB's key/value pairs in key order.
+//
+// When encouter an error any 'seeks method' will return false and will
+// yield no key/value pairs. The error can be queried by calling the Error
+// method. Calling Release is still necessary.
+//
+// An iterator must be released after use, but it is not necessary to read
+// an iterator until exhaustion.
+// Also, an iterator is not necessarily goroutine-safe, but it is safe to use
+// multiple iterators concurrently, with each in a dedicated goroutine.
+type Iterator interface {
+	CommonIterator
+
+	// Key returns the key of the current key/value pair, or nil if done.
+	// The caller should not modify the contents of the returned slice, and
+	// its contents may change on the next call to any 'seeks method'.
+	Key() []byte
+
+	// Value returns the key of the current key/value pair, or nil if done.
+	// The caller should not modify the contents of the returned slice, and
+	// its contents may change on the next call to any 'seeks method'.
+	Value() []byte
+}
+
+// ErrorCallbackSetter is the interface that wraps basic SetErrorCallback
+// method.
+//
+// ErrorCallbackSetter implemented by indexed and merged iterator.
+type ErrorCallbackSetter interface {
+	// SetErrorCallback allows set an error callback of the coresponding
+	// iterator. Use nil to clear the callback.
+	SetErrorCallback(f func(err error))
+}
+
+type emptyIterator struct {
+	releaser util.Releaser
+	released bool
+	err      error
+}
+
+func (i *emptyIterator) rErr() {
+	if i.err == nil && i.released {
+		i.err = errors.New("leveldb/iterator: iterator released")
+	}
+}
+
+func (i *emptyIterator) Release() {
+	if i.releaser != nil {
+		i.releaser.Release()
+		i.releaser = nil
+	}
+	i.released = true
+}
+
+func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
+	if !i.released {
+		i.releaser = releaser
+	}
+}
+
+func (*emptyIterator) Valid() bool            { return false }
+func (i *emptyIterator) First() bool          { i.rErr(); return false }
+func (i *emptyIterator) Last() bool           { i.rErr(); return false }
+func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false }
+func (i *emptyIterator) Next() bool           { i.rErr(); return false }
+func (i *emptyIterator) Prev() bool           { i.rErr(); return false }
+func (*emptyIterator) Key() []byte            { return nil }
+func (*emptyIterator) Value() []byte          { return nil }
+func (i *emptyIterator) Error() error         { return i.err }
+
+// NewEmptyIterator creates an empty iterator. The err parameter can be
+// nil, but if not nil the given err will be returned by Error method.
+func NewEmptyIterator(err error) Iterator {
+	return &emptyIterator{err: err}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go

@@ -0,0 +1,17 @@
+package iterator_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+func TestIterator(t *testing.T) {
+	testutil.RunDefer()
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Iterator Suite")
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go

@@ -0,0 +1,307 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator
+
+import (
+	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
+)
+
+type dir int
+
+const (
+	dirReleased dir = iota - 1
+	dirSOI
+	dirEOI
+	dirBackward
+	dirForward
+)
+
+type mergedIterator struct {
+	cmp    comparer.Comparer
+	iters  []Iterator
+	strict bool
+
+	keys     [][]byte
+	index    int
+	dir      dir
+	err      error
+	errf     func(err error)
+	releaser util.Releaser
+}
+
+func assertKey(key []byte) []byte {
+	if key == nil {
+		panic("leveldb/iterator: nil key")
+	}
+	return key
+}
+
+func (i *mergedIterator) iterErr(iter Iterator) bool {
+	if i.errf != nil {
+		if err := iter.Error(); err != nil {
+			i.errf(err)
+		}
+	}
+	if i.strict {
+		if err := iter.Error(); err != nil {
+			i.err = err
+			return true
+		}
+	}
+	return false
+}
+
+func (i *mergedIterator) Valid() bool {
+	return i.err == nil && i.dir > dirEOI
+}
+
+func (i *mergedIterator) First() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	for x, iter := range i.iters {
+		switch {
+		case iter.First():
+			i.keys[x] = assertKey(iter.Key())
+		case i.iterErr(iter):
+			return false
+		default:
+			i.keys[x] = nil
+		}
+	}
+	i.dir = dirSOI
+	return i.next()
+}
+
+func (i *mergedIterator) Last() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	for x, iter := range i.iters {
+		switch {
+		case iter.Last():
+			i.keys[x] = assertKey(iter.Key())
+		case i.iterErr(iter):
+			return false
+		default:
+			i.keys[x] = nil
+		}
+	}
+	i.dir = dirEOI
+	return i.prev()
+}
+
+func (i *mergedIterator) Seek(key []byte) bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	for x, iter := range i.iters {
+		switch {
+		case iter.Seek(key):
+			i.keys[x] = assertKey(iter.Key())
+		case i.iterErr(iter):
+			return false
+		default:
+			i.keys[x] = nil
+		}
+	}
+	i.dir = dirSOI
+	return i.next()
+}
+
+func (i *mergedIterator) next() bool {
+	var key []byte
+	if i.dir == dirForward {
+		key = i.keys[i.index]
+	}
+	for x, tkey := range i.keys {
+		if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) < 0) {
+			key = tkey
+			i.index = x
+		}
+	}
+	if key == nil {
+		i.dir = dirEOI
+		return false
+	}
+	i.dir = dirForward
+	return true
+}
+
+func (i *mergedIterator) Next() bool {
+	if i.dir == dirEOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	switch i.dir {
+	case dirSOI:
+		return i.First()
+	case dirBackward:
+		key := append([]byte{}, i.keys[i.index]...)
+		if !i.Seek(key) {
+			return false
+		}
+		return i.Next()
+	}
+
+	x := i.index
+	iter := i.iters[x]
+	switch {
+	case iter.Next():
+		i.keys[x] = assertKey(iter.Key())
+	case i.iterErr(iter):
+		return false
+	default:
+		i.keys[x] = nil
+	}
+	return i.next()
+}
+
+func (i *mergedIterator) prev() bool {
+	var key []byte
+	if i.dir == dirBackward {
+		key = i.keys[i.index]
+	}
+	for x, tkey := range i.keys {
+		if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) > 0) {
+			key = tkey
+			i.index = x
+		}
+	}
+	if key == nil {
+		i.dir = dirSOI
+		return false
+	}
+	i.dir = dirBackward
+	return true
+}
+
+func (i *mergedIterator) Prev() bool {
+	if i.dir == dirSOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	switch i.dir {
+	case dirEOI:
+		return i.Last()
+	case dirForward:
+		key := append([]byte{}, i.keys[i.index]...)
+		for x, iter := range i.iters {
+			if x == i.index {
+				continue
+			}
+			seek := iter.Seek(key)
+			switch {
+			case seek && iter.Prev(), !seek && iter.Last():
+				i.keys[x] = assertKey(iter.Key())
+			case i.iterErr(iter):
+				return false
+			default:
+				i.keys[x] = nil
+			}
+		}
+	}
+
+	x := i.index
+	iter := i.iters[x]
+	switch {
+	case iter.Prev():
+		i.keys[x] = assertKey(iter.Key())
+	case i.iterErr(iter):
+		return false
+	default:
+		i.keys[x] = nil
+	}
+	return i.prev()
+}
+
+func (i *mergedIterator) Key() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.keys[i.index]
+}
+
+func (i *mergedIterator) Value() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.iters[i.index].Value()
+}
+
+func (i *mergedIterator) Release() {
+	if i.dir != dirReleased {
+		i.dir = dirReleased
+		for _, iter := range i.iters {
+			iter.Release()
+		}
+		i.iters = nil
+		i.keys = nil
+		if i.releaser != nil {
+			i.releaser.Release()
+			i.releaser = nil
+		}
+	}
+}
+
+func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
+	if i.dir != dirReleased {
+		i.releaser = releaser
+	}
+}
+
+func (i *mergedIterator) Error() error {
+	return i.err
+}
+
+func (i *mergedIterator) SetErrorCallback(f func(err error)) {
+	i.errf = f
+}
+
+// NewMergedIterator returns an iterator that merges its input. Walking the
+// resultant iterator will return all key/value pairs of all input iterators
+// in strictly increasing key order, as defined by cmp.
+// The input's key ranges may overlap, but there are assumed to be no duplicate
+// keys: if iters[i] contains a key k then iters[j] will not contain that key k.
+// None of the iters may be nil.
+//
+// If strict is true then error yield by any iterators will halt the merged
+// iterator, on contrary if strict is false then the merged iterator will
+// ignore those error and move on to the next iterator.
+func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator {
+	return &mergedIterator{
+		iters:  iters,
+		cmp:    cmp,
+		strict: strict,
+		keys:   make([][]byte, len(iters)),
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package iterator_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	. "github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+var _ = testutil.Defer(func() {
+	Describe("Merged iterator", func() {
+		Test := func(filled int, empty int) func() {
+			return func() {
+				It("Should iterates and seeks correctly", func(done Done) {
+					rnd := testutil.NewRand()
+
+					// Build key/value.
+					filledKV := make([]testutil.KeyValue, filled)
+					kv := testutil.KeyValue_Generate(nil, 100, 1, 10, 4, 4)
+					kv.Iterate(func(i int, key, value []byte) {
+						filledKV[rnd.Intn(filled)].Put(key, value)
+					})
+
+					// Create itearators.
+					iters := make([]Iterator, filled+empty)
+					for i := range iters {
+						if empty == 0 || (rnd.Int()%2 == 0 && filled > 0) {
+							filled--
+							Expect(filledKV[filled].Len()).ShouldNot(BeZero())
+							iters[i] = NewArrayIterator(filledKV[filled])
+						} else {
+							empty--
+							iters[i] = NewEmptyIterator(nil)
+						}
+					}
+
+					// Test the iterator.
+					t := testutil.IteratorTesting{
+						KeyValue: kv.Clone(),
+						Iter:     NewMergedIterator(iters, comparer.DefaultComparer, true),
+					}
+					testutil.DoIteratorTesting(&t)
+					done <- true
+				}, 1.5)
+			}
+		}
+
+		Describe("with three, all filled iterators", Test(3, 0))
+		Describe("with one filled, one empty iterators", Test(1, 1))
+		Describe("with one filled, two empty iterators", Test(1, 2))
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go

@@ -0,0 +1,513 @@
+// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record.go?r=1d5ccbe03246da926391ee12d1c6caae054ff4b0
+// License, authors and contributors informations can be found at bellow URLs respectively:
+// 	https://code.google.com/p/leveldb-go/source/browse/LICENSE
+//	https://code.google.com/p/leveldb-go/source/browse/AUTHORS
+//  https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
+
+// Package journal reads and writes sequences of journals. Each journal is a stream
+// of bytes that completes before the next journal starts.
+//
+// When reading, call Next to obtain an io.Reader for the next journal. Next will
+// return io.EOF when there are no more journals. It is valid to call Next
+// without reading the current journal to exhaustion.
+//
+// When writing, call Next to obtain an io.Writer for the next journal. Calling
+// Next finishes the current journal. Call Close to finish the final journal.
+//
+// Optionally, call Flush to finish the current journal and flush the underlying
+// writer without starting a new journal. To start a new journal after flushing,
+// call Next.
+//
+// Neither Readers or Writers are safe to use concurrently.
+//
+// Example code:
+//	func read(r io.Reader) ([]string, error) {
+//		var ss []string
+//		journals := journal.NewReader(r, nil, true, true)
+//		for {
+//			j, err := journals.Next()
+//			if err == io.EOF {
+//				break
+//			}
+//			if err != nil {
+//				return nil, err
+//			}
+//			s, err := ioutil.ReadAll(j)
+//			if err != nil {
+//				return nil, err
+//			}
+//			ss = append(ss, string(s))
+//		}
+//		return ss, nil
+//	}
+//
+//	func write(w io.Writer, ss []string) error {
+//		journals := journal.NewWriter(w)
+//		for _, s := range ss {
+//			j, err := journals.Next()
+//			if err != nil {
+//				return err
+//			}
+//			if _, err := j.Write([]byte(s)), err != nil {
+//				return err
+//			}
+//		}
+//		return journals.Close()
+//	}
+//
+// The wire format is that the stream is divided into 32KiB blocks, and each
+// block contains a number of tightly packed chunks. Chunks cannot cross block
+// boundaries. The last block may be shorter than 32 KiB. Any unused bytes in a
+// block must be zero.
+//
+// A journal maps to one or more chunks. Each chunk has a 7 byte header (a 4
+// byte checksum, a 2 byte little-endian uint16 length, and a 1 byte chunk type)
+// followed by a payload. The checksum is over the chunk type and the payload.
+//
+// There are four chunk types: whether the chunk is the full journal, or the
+// first, middle or last chunk of a multi-chunk journal. A multi-chunk journal
+// has one first chunk, zero or more middle chunks, and one last chunk.
+//
+// The wire format allows for limited recovery in the face of data corruption:
+// on a format error (such as a checksum mismatch), the reader moves to the
+// next block and looks for the next full or first chunk.
+package journal
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// These constants are part of the wire format and should not be changed.
+const (
+	fullChunkType   = 1
+	firstChunkType  = 2
+	middleChunkType = 3
+	lastChunkType   = 4
+)
+
+const (
+	blockSize  = 32 * 1024
+	headerSize = 7
+)
+
+type flusher interface {
+	Flush() error
+}
+
+// DroppedError is the error type that passed to Dropper.Drop method.
+type DroppedError struct {
+	Size   int
+	Reason string
+}
+
+func (e DroppedError) Error() string {
+	return fmt.Sprintf("leveldb/journal: dropped %d bytes: %s", e.Size, e.Reason)
+}
+
+// Dropper is the interface that wrap simple Drop method. The Drop
+// method will be called when the journal reader dropping a chunk.
+type Dropper interface {
+	Drop(err error)
+}
+
+// Reader reads journals from an underlying io.Reader.
+type Reader struct {
+	// r is the underlying reader.
+	r io.Reader
+	// the dropper.
+	dropper Dropper
+	// strict flag.
+	strict bool
+	// checksum flag.
+	checksum bool
+	// seq is the sequence number of the current journal.
+	seq int
+	// buf[i:j] is the unread portion of the current chunk's payload.
+	// The low bound, i, excludes the chunk header.
+	i, j int
+	// n is the number of bytes of buf that are valid. Once reading has started,
+	// only the final block can have n < blockSize.
+	n int
+	// last is whether the current chunk is the last chunk of the journal.
+	last bool
+	// err is any accumulated error.
+	err error
+	// buf is the buffer.
+	buf [blockSize]byte
+}
+
+// NewReader returns a new reader. The dropper may be nil, and if
+// strict is true then corrupted or invalid chunk will halt the journal
+// reader entirely.
+func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
+	return &Reader{
+		r:        r,
+		dropper:  dropper,
+		strict:   strict,
+		checksum: checksum,
+		last:     true,
+	}
+}
+
+// nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
+// next block into the buffer if necessary.
+func (r *Reader) nextChunk(wantFirst, skip bool) error {
+	for {
+		if r.j+headerSize <= r.n {
+			checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
+			length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
+			chunkType := r.buf[r.j+6]
+
+			var err error
+			if checksum == 0 && length == 0 && chunkType == 0 {
+				// Drop entire block.
+				err = DroppedError{r.n - r.j, "zero header"}
+				r.i = r.n
+				r.j = r.n
+			} else {
+				m := r.n - r.j
+				r.i = r.j + headerSize
+				r.j = r.j + headerSize + int(length)
+				if r.j > r.n {
+					// Drop entire block.
+					err = DroppedError{m, "chunk length overflows block"}
+					r.i = r.n
+					r.j = r.n
+				} else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
+					// Drop entire block.
+					err = DroppedError{m, "checksum mismatch"}
+					r.i = r.n
+					r.j = r.n
+				}
+			}
+			if wantFirst && err == nil && chunkType != fullChunkType && chunkType != firstChunkType {
+				if skip {
+					// The chunk are intentionally skipped.
+					if chunkType == lastChunkType {
+						skip = false
+					}
+					continue
+				} else {
+					// Drop the chunk.
+					err = DroppedError{r.j - r.i + headerSize, "orphan chunk"}
+				}
+			}
+			if err == nil {
+				r.last = chunkType == fullChunkType || chunkType == lastChunkType
+			} else {
+				if r.dropper != nil {
+					r.dropper.Drop(err)
+				}
+				if r.strict {
+					r.err = err
+				}
+			}
+			return err
+		}
+		if r.n < blockSize && r.n > 0 {
+			// This is the last block.
+			if r.j != r.n {
+				r.err = io.ErrUnexpectedEOF
+			} else {
+				r.err = io.EOF
+			}
+			return r.err
+		}
+		n, err := io.ReadFull(r.r, r.buf[:])
+		if err != nil && err != io.ErrUnexpectedEOF {
+			r.err = err
+			return r.err
+		}
+		if n == 0 {
+			r.err = io.EOF
+			return r.err
+		}
+		r.i, r.j, r.n = 0, 0, n
+	}
+}
+
+// Next returns a reader for the next journal. It returns io.EOF if there are no
+// more journals. The reader returned becomes stale after the next Next call,
+// and should no longer be used.
+func (r *Reader) Next() (io.Reader, error) {
+	r.seq++
+	if r.err != nil {
+		return nil, r.err
+	}
+	skip := !r.last
+	for {
+		r.i = r.j
+		if r.nextChunk(true, skip) != nil {
+			// So that 'orphan chunk' drop will be reported.
+			skip = false
+		} else {
+			break
+		}
+		if r.err != nil {
+			return nil, r.err
+		}
+	}
+	return &singleReader{r, r.seq, nil}, nil
+}
+
+// Reset resets the journal reader, allows reuse of the journal reader.
+func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
+	r.seq++
+	err := r.err
+	r.r = reader
+	r.dropper = dropper
+	r.strict = strict
+	r.checksum = checksum
+	r.i = 0
+	r.j = 0
+	r.n = 0
+	r.last = true
+	r.err = nil
+	return err
+}
+
+type singleReader struct {
+	r   *Reader
+	seq int
+	err error
+}
+
+func (x *singleReader) Read(p []byte) (int, error) {
+	r := x.r
+	if r.seq != x.seq {
+		return 0, errors.New("leveldb/journal: stale reader")
+	}
+	if x.err != nil {
+		return 0, x.err
+	}
+	if r.err != nil {
+		return 0, r.err
+	}
+	for r.i == r.j {
+		if r.last {
+			return 0, io.EOF
+		}
+		if x.err = r.nextChunk(false, false); x.err != nil {
+			return 0, x.err
+		}
+	}
+	n := copy(p, r.buf[r.i:r.j])
+	r.i += n
+	return n, nil
+}
+
+func (x *singleReader) ReadByte() (byte, error) {
+	r := x.r
+	if r.seq != x.seq {
+		return 0, errors.New("leveldb/journal: stale reader")
+	}
+	if x.err != nil {
+		return 0, x.err
+	}
+	if r.err != nil {
+		return 0, r.err
+	}
+	for r.i == r.j {
+		if r.last {
+			return 0, io.EOF
+		}
+		if x.err = r.nextChunk(false, false); x.err != nil {
+			return 0, x.err
+		}
+	}
+	c := r.buf[r.i]
+	r.i++
+	return c, nil
+}
+
+// Writer writes journals to an underlying io.Writer.
+type Writer struct {
+	// w is the underlying writer.
+	w io.Writer
+	// seq is the sequence number of the current journal.
+	seq int
+	// f is w as a flusher.
+	f flusher
+	// buf[i:j] is the bytes that will become the current chunk.
+	// The low bound, i, includes the chunk header.
+	i, j int
+	// buf[:written] has already been written to w.
+	// written is zero unless Flush has been called.
+	written int
+	// first is whether the current chunk is the first chunk of the journal.
+	first bool
+	// pending is whether a chunk is buffered but not yet written.
+	pending bool
+	// err is any accumulated error.
+	err error
+	// buf is the buffer.
+	buf [blockSize]byte
+}
+
+// NewWriter returns a new Writer.
+func NewWriter(w io.Writer) *Writer {
+	f, _ := w.(flusher)
+	return &Writer{
+		w: w,
+		f: f,
+	}
+}
+
+// fillHeader fills in the header for the pending chunk.
+func (w *Writer) fillHeader(last bool) {
+	if w.i+headerSize > w.j || w.j > blockSize {
+		panic("leveldb/journal: bad writer state")
+	}
+	if last {
+		if w.first {
+			w.buf[w.i+6] = fullChunkType
+		} else {
+			w.buf[w.i+6] = lastChunkType
+		}
+	} else {
+		if w.first {
+			w.buf[w.i+6] = firstChunkType
+		} else {
+			w.buf[w.i+6] = middleChunkType
+		}
+	}
+	binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], util.NewCRC(w.buf[w.i+6:w.j]).Value())
+	binary.LittleEndian.PutUint16(w.buf[w.i+4:w.i+6], uint16(w.j-w.i-headerSize))
+}
+
+// writeBlock writes the buffered block to the underlying writer, and reserves
+// space for the next chunk's header.
+func (w *Writer) writeBlock() {
+	_, w.err = w.w.Write(w.buf[w.written:])
+	w.i = 0
+	w.j = headerSize
+	w.written = 0
+}
+
+// writePending finishes the current journal and writes the buffer to the
+// underlying writer.
+func (w *Writer) writePending() {
+	if w.err != nil {
+		return
+	}
+	if w.pending {
+		w.fillHeader(true)
+		w.pending = false
+	}
+	_, w.err = w.w.Write(w.buf[w.written:w.j])
+	w.written = w.j
+}
+
+// Close finishes the current journal and closes the writer.
+func (w *Writer) Close() error {
+	w.seq++
+	w.writePending()
+	if w.err != nil {
+		return w.err
+	}
+	w.err = errors.New("leveldb/journal: closed Writer")
+	return nil
+}
+
+// Flush finishes the current journal, writes to the underlying writer, and
+// flushes it if that writer implements interface{ Flush() error }.
+func (w *Writer) Flush() error {
+	w.seq++
+	w.writePending()
+	if w.err != nil {
+		return w.err
+	}
+	if w.f != nil {
+		w.err = w.f.Flush()
+		return w.err
+	}
+	return nil
+}
+
+// Reset resets the journal writer, allows reuse of the journal writer. Reset
+// will also closes the journal writer if not already.
+func (w *Writer) Reset(writer io.Writer) (err error) {
+	w.seq++
+	if w.err == nil {
+		w.writePending()
+		err = w.err
+	}
+	w.w = writer
+	w.f, _ = writer.(flusher)
+	w.i = 0
+	w.j = 0
+	w.written = 0
+	w.first = false
+	w.pending = false
+	w.err = nil
+	return
+}
+
+// Next returns a writer for the next journal. The writer returned becomes stale
+// after the next Close, Flush or Next call, and should no longer be used.
+func (w *Writer) Next() (io.Writer, error) {
+	w.seq++
+	if w.err != nil {
+		return nil, w.err
+	}
+	if w.pending {
+		w.fillHeader(true)
+	}
+	w.i = w.j
+	w.j = w.j + headerSize
+	// Check if there is room in the block for the header.
+	if w.j > blockSize {
+		// Fill in the rest of the block with zeroes.
+		for k := w.i; k < blockSize; k++ {
+			w.buf[k] = 0
+		}
+		w.writeBlock()
+		if w.err != nil {
+			return nil, w.err
+		}
+	}
+	w.first = true
+	w.pending = true
+	return singleWriter{w, w.seq}, nil
+}
+
+type singleWriter struct {
+	w   *Writer
+	seq int
+}
+
+func (x singleWriter) Write(p []byte) (int, error) {
+	w := x.w
+	if w.seq != x.seq {
+		return 0, errors.New("leveldb/journal: stale writer")
+	}
+	if w.err != nil {
+		return 0, w.err
+	}
+	n0 := len(p)
+	for len(p) > 0 {
+		// Write a block, if it is full.
+		if w.j == blockSize {
+			w.fillHeader(false)
+			w.writeBlock()
+			if w.err != nil {
+				return 0, w.err
+			}
+			w.first = false
+		}
+		// Copy bytes into the buffer.
+		n := copy(w.buf[w.j:], p)
+		w.j += n
+		p = p[n:]
+	}
+	return n0, nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go

@@ -0,0 +1,328 @@
+// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record_test.go?r=df1fa28f7f3be6c3935548169002309c12967135
+// License, authors and contributors informations can be found at bellow URLs respectively:
+// 	https://code.google.com/p/leveldb-go/source/browse/LICENSE
+//	https://code.google.com/p/leveldb-go/source/browse/AUTHORS
+//  https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
+
+package journal
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"strings"
+	"testing"
+)
+
+type dropper struct {
+	t *testing.T
+}
+
+func (d dropper) Drop(err error) {
+	d.t.Log(err)
+}
+
+func short(s string) string {
+	if len(s) < 64 {
+		return s
+	}
+	return fmt.Sprintf("%s...(skipping %d bytes)...%s", s[:20], len(s)-40, s[len(s)-20:])
+}
+
+// big returns a string of length n, composed of repetitions of partial.
+func big(partial string, n int) string {
+	return strings.Repeat(partial, n/len(partial)+1)[:n]
+}
+
+func TestEmpty(t *testing.T) {
+	buf := new(bytes.Buffer)
+	r := NewReader(buf, dropper{t}, true, true)
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("got %v, want %v", err, io.EOF)
+	}
+}
+
+func testGenerator(t *testing.T, reset func(), gen func() (string, bool)) {
+	buf := new(bytes.Buffer)
+
+	reset()
+	w := NewWriter(buf)
+	for {
+		s, ok := gen()
+		if !ok {
+			break
+		}
+		ww, err := w.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := ww.Write([]byte(s)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reset()
+	r := NewReader(buf, dropper{t}, true, true)
+	for {
+		s, ok := gen()
+		if !ok {
+			break
+		}
+		rr, err := r.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+		x, err := ioutil.ReadAll(rr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(x) != s {
+			t.Fatalf("got %q, want %q", short(string(x)), short(s))
+		}
+	}
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("got %v, want %v", err, io.EOF)
+	}
+}
+
+func testLiterals(t *testing.T, s []string) {
+	var i int
+	reset := func() {
+		i = 0
+	}
+	gen := func() (string, bool) {
+		if i == len(s) {
+			return "", false
+		}
+		i++
+		return s[i-1], true
+	}
+	testGenerator(t, reset, gen)
+}
+
+func TestMany(t *testing.T) {
+	const n = 1e5
+	var i int
+	reset := func() {
+		i = 0
+	}
+	gen := func() (string, bool) {
+		if i == n {
+			return "", false
+		}
+		i++
+		return fmt.Sprintf("%d.", i-1), true
+	}
+	testGenerator(t, reset, gen)
+}
+
+func TestRandom(t *testing.T) {
+	const n = 1e2
+	var (
+		i int
+		r *rand.Rand
+	)
+	reset := func() {
+		i, r = 0, rand.New(rand.NewSource(0))
+	}
+	gen := func() (string, bool) {
+		if i == n {
+			return "", false
+		}
+		i++
+		return strings.Repeat(string(uint8(i)), r.Intn(2*blockSize+16)), true
+	}
+	testGenerator(t, reset, gen)
+}
+
+func TestBasic(t *testing.T) {
+	testLiterals(t, []string{
+		strings.Repeat("a", 1000),
+		strings.Repeat("b", 97270),
+		strings.Repeat("c", 8000),
+	})
+}
+
+func TestBoundary(t *testing.T) {
+	for i := blockSize - 16; i < blockSize+16; i++ {
+		s0 := big("abcd", i)
+		for j := blockSize - 16; j < blockSize+16; j++ {
+			s1 := big("ABCDE", j)
+			testLiterals(t, []string{s0, s1})
+			testLiterals(t, []string{s0, "", s1})
+			testLiterals(t, []string{s0, "x", s1})
+		}
+	}
+}
+
+func TestFlush(t *testing.T) {
+	buf := new(bytes.Buffer)
+	w := NewWriter(buf)
+	// Write a couple of records. Everything should still be held
+	// in the record.Writer buffer, so that buf.Len should be 0.
+	w0, _ := w.Next()
+	w0.Write([]byte("0"))
+	w1, _ := w.Next()
+	w1.Write([]byte("11"))
+	if got, want := buf.Len(), 0; got != want {
+		t.Fatalf("buffer length #0: got %d want %d", got, want)
+	}
+	// Flush the record.Writer buffer, which should yield 17 bytes.
+	// 17 = 2*7 + 1 + 2, which is two headers and 1 + 2 payload bytes.
+	if err := w.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := buf.Len(), 17; got != want {
+		t.Fatalf("buffer length #1: got %d want %d", got, want)
+	}
+	// Do another write, one that isn't large enough to complete the block.
+	// The write should not have flowed through to buf.
+	w2, _ := w.Next()
+	w2.Write(bytes.Repeat([]byte("2"), 10000))
+	if got, want := buf.Len(), 17; got != want {
+		t.Fatalf("buffer length #2: got %d want %d", got, want)
+	}
+	// Flushing should get us up to 10024 bytes written.
+	// 10024 = 17 + 7 + 10000.
+	if err := w.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := buf.Len(), 10024; got != want {
+		t.Fatalf("buffer length #3: got %d want %d", got, want)
+	}
+	// Do a bigger write, one that completes the current block.
+	// We should now have 32768 bytes (a complete block), without
+	// an explicit flush.
+	w3, _ := w.Next()
+	w3.Write(bytes.Repeat([]byte("3"), 40000))
+	if got, want := buf.Len(), 32768; got != want {
+		t.Fatalf("buffer length #4: got %d want %d", got, want)
+	}
+	// Flushing should get us up to 50038 bytes written.
+	// 50038 = 10024 + 2*7 + 40000. There are two headers because
+	// the one record was split into two chunks.
+	if err := w.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := buf.Len(), 50038; got != want {
+		t.Fatalf("buffer length #5: got %d want %d", got, want)
+	}
+	// Check that reading those records give the right lengths.
+	r := NewReader(buf, dropper{t}, true, true)
+	wants := []int64{1, 2, 10000, 40000}
+	for i, want := range wants {
+		rr, _ := r.Next()
+		n, err := io.Copy(ioutil.Discard, rr)
+		if err != nil {
+			t.Fatalf("read #%d: %v", i, err)
+		}
+		if n != want {
+			t.Fatalf("read #%d: got %d bytes want %d", i, n, want)
+		}
+	}
+}
+
+func TestNonExhaustiveRead(t *testing.T) {
+	const n = 100
+	buf := new(bytes.Buffer)
+	p := make([]byte, 10)
+	rnd := rand.New(rand.NewSource(1))
+
+	w := NewWriter(buf)
+	for i := 0; i < n; i++ {
+		length := len(p) + rnd.Intn(3*blockSize)
+		s := string(uint8(i)) + "123456789abcdefgh"
+		ww, _ := w.Next()
+		ww.Write([]byte(big(s, length)))
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r := NewReader(buf, dropper{t}, true, true)
+	for i := 0; i < n; i++ {
+		rr, _ := r.Next()
+		_, err := io.ReadFull(rr, p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := string(uint8(i)) + "123456789"
+		if got := string(p); got != want {
+			t.Fatalf("read #%d: got %q want %q", i, got, want)
+		}
+	}
+}
+
+func TestStaleReader(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+	w0, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	w0.Write([]byte("0"))
+	w1, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	w1.Write([]byte("11"))
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r := NewReader(buf, dropper{t}, true, true)
+	r0, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	r1, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := make([]byte, 1)
+	if _, err := r0.Read(p); err == nil || !strings.Contains(err.Error(), "stale") {
+		t.Fatalf("stale read #0: unexpected error: %v", err)
+	}
+	if _, err := r1.Read(p); err != nil {
+		t.Fatalf("fresh read #1: got %v want nil error", err)
+	}
+	if p[0] != '1' {
+		t.Fatalf("fresh read #1: byte contents: got '%c' want '1'", p[0])
+	}
+}
+
+func TestStaleWriter(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+	w0, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	w1, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w0.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
+		t.Fatalf("stale write #0: unexpected error: %v", err)
+	}
+	if _, err := w1.Write([]byte("11")); err != nil {
+		t.Fatalf("fresh write #1: got %v want nil error", err)
+	}
+	if err := w.Flush(); err != nil {
+		t.Fatalf("flush: %v", err)
+	}
+	if _, err := w1.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
+		t.Fatalf("stale write #1: unexpected error: %v", err)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go

@@ -0,0 +1,139 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type vType int
+
+func (t vType) String() string {
+	switch t {
+	case tDel:
+		return "d"
+	case tVal:
+		return "v"
+	}
+	return "x"
+}
+
+// Value types encoded as the last component of internal keys.
+// Don't modify; this value are saved to disk.
+const (
+	tDel vType = iota
+	tVal
+)
+
+// tSeek defines the vType that should be passed when constructing an
+// internal key for seeking to a particular sequence number (since we
+// sort sequence numbers in decreasing order and the value type is
+// embedded as the low 8 bits in the sequence number in internal keys,
+// we need to use the highest-numbered ValueType, not the lowest).
+const tSeek = tVal
+
+const (
+	// Maximum value possible for sequence number; the 8-bits are
+	// used by value type, so its can packed together in single
+	// 64-bit integer.
+	kMaxSeq uint64 = (uint64(1) << 56) - 1
+	// Maximum value possible for packed sequence number and type.
+	kMaxNum uint64 = (kMaxSeq << 8) | uint64(tSeek)
+)
+
+// Maximum number encoded in bytes.
+var kMaxNumBytes = make([]byte, 8)
+
+func init() {
+	binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum)
+}
+
+type iKey []byte
+
+func newIKey(ukey []byte, seq uint64, t vType) iKey {
+	if seq > kMaxSeq || t > tVal {
+		panic("invalid seq number or value type")
+	}
+
+	b := make(iKey, len(ukey)+8)
+	copy(b, ukey)
+	binary.LittleEndian.PutUint64(b[len(ukey):], (seq<<8)|uint64(t))
+	return b
+}
+
+func parseIkey(p []byte) (ukey []byte, seq uint64, t vType, ok bool) {
+	if len(p) < 8 {
+		return
+	}
+	num := binary.LittleEndian.Uint64(p[len(p)-8:])
+	seq, t = uint64(num>>8), vType(num&0xff)
+	if t > tVal {
+		return
+	}
+	ukey = p[:len(p)-8]
+	ok = true
+	return
+}
+
+func validIkey(p []byte) bool {
+	_, _, _, ok := parseIkey(p)
+	return ok
+}
+
+func (p iKey) assert() {
+	if p == nil {
+		panic("nil iKey")
+	}
+	if len(p) < 8 {
+		panic(fmt.Sprintf("invalid iKey %q, len=%d", []byte(p), len(p)))
+	}
+}
+
+func (p iKey) ok() bool {
+	if len(p) < 8 {
+		return false
+	}
+	_, _, ok := p.parseNum()
+	return ok
+}
+
+func (p iKey) ukey() []byte {
+	p.assert()
+	return p[:len(p)-8]
+}
+
+func (p iKey) num() uint64 {
+	p.assert()
+	return binary.LittleEndian.Uint64(p[len(p)-8:])
+}
+
+func (p iKey) parseNum() (seq uint64, t vType, ok bool) {
+	if p == nil {
+		panic("nil iKey")
+	}
+	if len(p) < 8 {
+		return
+	}
+	num := p.num()
+	seq, t = uint64(num>>8), vType(num&0xff)
+	if t > tVal {
+		return 0, 0, false
+	}
+	ok = true
+	return
+}
+
+func (p iKey) String() string {
+	if len(p) == 0 {
+		return "<nil>"
+	}
+	if seq, t, ok := p.parseNum(); ok {
+		return fmt.Sprintf("%s,%s%d", shorten(string(p.ukey())), t, seq)
+	}
+	return "<invalid>"
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+)
+
+var icmp = &iComparer{comparer.DefaultComparer}
+
+func ikey(key string, seq uint64, t vType) iKey {
+	return newIKey([]byte(key), uint64(seq), t)
+}
+
+func shortSep(a, b []byte) []byte {
+	dst := make([]byte, len(a))
+	dst = icmp.Separator(dst[:0], a, b)
+	if dst == nil {
+		return a
+	}
+	return dst
+}
+
+func shortSuccessor(b []byte) []byte {
+	dst := make([]byte, len(b))
+	dst = icmp.Successor(dst[:0], b)
+	if dst == nil {
+		return b
+	}
+	return dst
+}
+
+func testSingleKey(t *testing.T, key string, seq uint64, vt vType) {
+	ik := ikey(key, seq, vt)
+
+	if !bytes.Equal(ik.ukey(), []byte(key)) {
+		t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
+	}
+
+	if rseq, rt, ok := ik.parseNum(); ok {
+		if rseq != seq {
+			t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
+		}
+
+		if rt != vt {
+			t.Errorf("type does not equal, got %v, want %v", rt, vt)
+		}
+	} else {
+		t.Error("cannot parse seq and type")
+	}
+}
+
+func TestIKey_EncodeDecode(t *testing.T) {
+	keys := []string{"", "k", "hello", "longggggggggggggggggggggg"}
+	seqs := []uint64{
+		1, 2, 3,
+		(1 << 8) - 1, 1 << 8, (1 << 8) + 1,
+		(1 << 16) - 1, 1 << 16, (1 << 16) + 1,
+		(1 << 32) - 1, 1 << 32, (1 << 32) + 1,
+	}
+	for _, key := range keys {
+		for _, seq := range seqs {
+			testSingleKey(t, key, seq, tVal)
+			testSingleKey(t, "hello", 1, tDel)
+		}
+	}
+}
+
+func assertBytes(t *testing.T, want, got []byte) {
+	if !bytes.Equal(got, want) {
+		t.Errorf("assert failed, got %v, want %v", got, want)
+	}
+}
+
+func TestIKeyShortSeparator(t *testing.T) {
+	// When user keys are same
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("foo", 99, tVal)))
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("foo", 101, tVal)))
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("foo", 100, tVal)))
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("foo", 100, tDel)))
+
+	// When user keys are misordered
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("bar", 99, tVal)))
+
+	// When user keys are different, but correctly ordered
+	assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("hello", 200, tVal)))
+
+	// When start user key is prefix of limit user key
+	assertBytes(t, ikey("foo", 100, tVal),
+		shortSep(ikey("foo", 100, tVal),
+			ikey("foobar", 200, tVal)))
+
+	// When limit user key is prefix of start user key
+	assertBytes(t, ikey("foobar", 100, tVal),
+		shortSep(ikey("foobar", 100, tVal),
+			ikey("foo", 200, tVal)))
+}
+
+func TestIKeyShortestSuccessor(t *testing.T) {
+	assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
+		shortSuccessor(ikey("foo", 100, tVal)))
+	assertBytes(t, ikey("\xff\xff", 100, tVal),
+		shortSuccessor(ikey("\xff\xff", 100, tVal)))
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go

@@ -0,0 +1,20 @@
+package leveldb
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+func TestLeveldb(t *testing.T) {
+	testutil.RunDefer()
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Leveldb Suite")
+
+	RegisterTestingT(t)
+	testutil.RunDefer("teardown")
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package memdb
+
+import (
+	"encoding/binary"
+	"math/rand"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+)
+
+func BenchmarkPut(b *testing.B) {
+	buf := make([][4]byte, b.N)
+	for i := range buf {
+		binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
+	}
+
+	b.ResetTimer()
+	p := New(comparer.DefaultComparer, 0)
+	for i := range buf {
+		p.Put(buf[i][:], nil)
+	}
+}
+
+func BenchmarkPutRandom(b *testing.B) {
+	buf := make([][4]byte, b.N)
+	for i := range buf {
+		binary.LittleEndian.PutUint32(buf[i][:], uint32(rand.Int()))
+	}
+
+	b.ResetTimer()
+	p := New(comparer.DefaultComparer, 0)
+	for i := range buf {
+		p.Put(buf[i][:], nil)
+	}
+}
+
+func BenchmarkGet(b *testing.B) {
+	buf := make([][4]byte, b.N)
+	for i := range buf {
+		binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
+	}
+
+	p := New(comparer.DefaultComparer, 0)
+	for i := range buf {
+		p.Put(buf[i][:], nil)
+	}
+
+	b.ResetTimer()
+	for i := range buf {
+		p.Get(buf[i][:])
+	}
+}
+
+func BenchmarkGetRandom(b *testing.B) {
+	buf := make([][4]byte, b.N)
+	for i := range buf {
+		binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
+	}
+
+	p := New(comparer.DefaultComparer, 0)
+	for i := range buf {
+		p.Put(buf[i][:], nil)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.Get(buf[rand.Int()%b.N][:])
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

@@ -0,0 +1,450 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package memdb provides in-memory key/value database implementation.
+package memdb
+
+import (
+	"math/rand"
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	ErrNotFound = util.ErrNotFound
+)
+
+const tMaxHeight = 12
+
+type dbIter struct {
+	util.BasicReleaser
+	p          *DB
+	slice      *util.Range
+	node       int
+	forward    bool
+	key, value []byte
+}
+
+func (i *dbIter) fill(checkStart, checkLimit bool) bool {
+	if i.node != 0 {
+		n := i.p.nodeData[i.node]
+		m := n + i.p.nodeData[i.node+nKey]
+		i.key = i.p.kvData[n:m]
+		if i.slice != nil {
+			switch {
+			case checkLimit && i.slice.Limit != nil && i.p.cmp.Compare(i.key, i.slice.Limit) >= 0:
+				fallthrough
+			case checkStart && i.slice.Start != nil && i.p.cmp.Compare(i.key, i.slice.Start) < 0:
+				i.node = 0
+				goto bail
+			}
+		}
+		i.value = i.p.kvData[m : m+i.p.nodeData[i.node+nVal]]
+		return true
+	}
+bail:
+	i.key = nil
+	i.value = nil
+	return false
+}
+
+func (i *dbIter) Valid() bool {
+	return i.node != 0
+}
+
+func (i *dbIter) First() bool {
+	i.forward = true
+	i.p.mu.RLock()
+	defer i.p.mu.RUnlock()
+	if i.slice != nil && i.slice.Start != nil {
+		i.node, _ = i.p.findGE(i.slice.Start, false)
+	} else {
+		i.node = i.p.nodeData[nNext]
+	}
+	return i.fill(false, true)
+}
+
+func (i *dbIter) Last() bool {
+	if i.p == nil {
+		return false
+	}
+	i.forward = false
+	i.p.mu.RLock()
+	defer i.p.mu.RUnlock()
+	if i.slice != nil && i.slice.Limit != nil {
+		i.node = i.p.findLT(i.slice.Limit)
+	} else {
+		i.node = i.p.findLast()
+	}
+	return i.fill(true, false)
+}
+
+func (i *dbIter) Seek(key []byte) bool {
+	if i.p == nil {
+		return false
+	}
+	i.forward = true
+	i.p.mu.RLock()
+	defer i.p.mu.RUnlock()
+	if i.slice != nil && i.slice.Start != nil && i.p.cmp.Compare(key, i.slice.Start) < 0 {
+		key = i.slice.Start
+	}
+	i.node, _ = i.p.findGE(key, false)
+	return i.fill(false, true)
+}
+
+func (i *dbIter) Next() bool {
+	if i.p == nil {
+		return false
+	}
+	if i.node == 0 {
+		if !i.forward {
+			return i.First()
+		}
+		return false
+	}
+	i.forward = true
+	i.p.mu.RLock()
+	defer i.p.mu.RUnlock()
+	i.node = i.p.nodeData[i.node+nNext]
+	return i.fill(false, true)
+}
+
+func (i *dbIter) Prev() bool {
+	if i.p == nil {
+		return false
+	}
+	if i.node == 0 {
+		if i.forward {
+			return i.Last()
+		}
+		return false
+	}
+	i.forward = false
+	i.p.mu.RLock()
+	defer i.p.mu.RUnlock()
+	i.node = i.p.findLT(i.key)
+	return i.fill(true, false)
+}
+
+func (i *dbIter) Key() []byte {
+	return i.key
+}
+
+func (i *dbIter) Value() []byte {
+	return i.value
+}
+
+func (i *dbIter) Error() error { return nil }
+
+func (i *dbIter) Release() {
+	if i.p != nil {
+		i.p = nil
+		i.node = 0
+		i.key = nil
+		i.value = nil
+		i.BasicReleaser.Release()
+	}
+}
+
+const (
+	nKV = iota
+	nKey
+	nVal
+	nHeight
+	nNext
+)
+
+// DB is an in-memory key/value database.
+type DB struct {
+	cmp comparer.BasicComparer
+	rnd *rand.Rand
+
+	mu     sync.RWMutex
+	kvData []byte
+	// Node data:
+	// [0]         : KV offset
+	// [1]         : Key length
+	// [2]         : Value length
+	// [3]         : Height
+	// [3..height] : Next nodes
+	nodeData  []int
+	prevNode  [tMaxHeight]int
+	maxHeight int
+	n         int
+	kvSize    int
+}
+
+func (p *DB) randHeight() (h int) {
+	const branching = 4
+	h = 1
+	for h < tMaxHeight && p.rnd.Int()%branching == 0 {
+		h++
+	}
+	return
+}
+
+func (p *DB) findGE(key []byte, prev bool) (int, bool) {
+	node := 0
+	h := p.maxHeight - 1
+	for {
+		next := p.nodeData[node+nNext+h]
+		cmp := 1
+		if next != 0 {
+			o := p.nodeData[next]
+			cmp = p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key)
+		}
+		if cmp < 0 {
+			// Keep searching in this list
+			node = next
+		} else {
+			if prev {
+				p.prevNode[h] = node
+			} else if cmp == 0 {
+				return next, true
+			}
+			if h == 0 {
+				return next, cmp == 0
+			}
+			h--
+		}
+	}
+}
+
+func (p *DB) findLT(key []byte) int {
+	node := 0
+	h := p.maxHeight - 1
+	for {
+		next := p.nodeData[node+nNext+h]
+		o := p.nodeData[next]
+		if next == 0 || p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) >= 0 {
+			if h == 0 {
+				break
+			}
+			h--
+		} else {
+			node = next
+		}
+	}
+	return node
+}
+
+func (p *DB) findLast() int {
+	node := 0
+	h := p.maxHeight - 1
+	for {
+		next := p.nodeData[node+nNext+h]
+		if next == 0 {
+			if h == 0 {
+				break
+			}
+			h--
+		} else {
+			node = next
+		}
+	}
+	return node
+}
+
+// Put sets the value for the given key. It overwrites any previous value
+// for that key; a DB is not a multi-map.
+//
+// It is safe to modify the contents of the arguments after Put returns.
+func (p *DB) Put(key []byte, value []byte) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if node, exact := p.findGE(key, true); exact {
+		kvOffset := len(p.kvData)
+		p.kvData = append(p.kvData, key...)
+		p.kvData = append(p.kvData, value...)
+		p.nodeData[node] = kvOffset
+		m := p.nodeData[node+nVal]
+		p.nodeData[node+nVal] = len(value)
+		p.kvSize += len(value) - m
+		return nil
+	}
+
+	h := p.randHeight()
+	if h > p.maxHeight {
+		for i := p.maxHeight; i < h; i++ {
+			p.prevNode[i] = 0
+		}
+		p.maxHeight = h
+	}
+
+	kvOffset := len(p.kvData)
+	p.kvData = append(p.kvData, key...)
+	p.kvData = append(p.kvData, value...)
+	// Node
+	node := len(p.nodeData)
+	p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h)
+	for i, n := range p.prevNode[:h] {
+		m := n + 4 + i
+		p.nodeData = append(p.nodeData, p.nodeData[m])
+		p.nodeData[m] = node
+	}
+
+	p.kvSize += len(key) + len(value)
+	p.n++
+	return nil
+}
+
+// Delete deletes the value for the given key. It returns ErrNotFound if
+// the DB does not contain the key.
+//
+// It is safe to modify the contents of the arguments after Delete returns.
+func (p *DB) Delete(key []byte) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	node, exact := p.findGE(key, true)
+	if !exact {
+		return ErrNotFound
+	}
+
+	h := p.nodeData[node+nHeight]
+	for i, n := range p.prevNode[:h] {
+		m := n + 4 + i
+		p.nodeData[m] = p.nodeData[p.nodeData[m]+nNext+i]
+	}
+
+	p.kvSize -= p.nodeData[node+nKey] + p.nodeData[node+nVal]
+	p.n--
+	return nil
+}
+
+// Contains returns true if the given key are in the DB.
+//
+// It is safe to modify the contents of the arguments after Contains returns.
+func (p *DB) Contains(key []byte) bool {
+	p.mu.RLock()
+	_, exact := p.findGE(key, false)
+	p.mu.RUnlock()
+	return exact
+}
+
+// Get gets the value for the given key. It returns error.ErrNotFound if the
+// DB does not contain the key.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Get returns.
+func (p *DB) Get(key []byte) (value []byte, err error) {
+	p.mu.RLock()
+	if node, exact := p.findGE(key, false); exact {
+		o := p.nodeData[node] + p.nodeData[node+nKey]
+		value = p.kvData[o : o+p.nodeData[node+nVal]]
+	} else {
+		err = ErrNotFound
+	}
+	p.mu.RUnlock()
+	return
+}
+
+// Find finds key/value pair whose key is greater than or equal to the
+// given key. It returns ErrNotFound if the table doesn't contain
+// such pair.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Find returns.
+func (p *DB) Find(key []byte) (rkey, value []byte, err error) {
+	p.mu.RLock()
+	if node, _ := p.findGE(key, false); node != 0 {
+		n := p.nodeData[node]
+		m := n + p.nodeData[node+nKey]
+		rkey = p.kvData[n:m]
+		value = p.kvData[m : m+p.nodeData[node+nVal]]
+	} else {
+		err = ErrNotFound
+	}
+	p.mu.RUnlock()
+	return
+}
+
+// NewIterator returns an iterator of the DB.
+// The returned iterator is not goroutine-safe, but it is safe to use
+// multiple iterators concurrently, with each in a dedicated goroutine.
+// It is also safe to use an iterator concurrently with modifying its
+// underlying DB. However, the resultant key/value pairs are not guaranteed
+// to be a consistent snapshot of the DB at a particular point in time.
+//
+// Slice allows slicing the iterator to only contains keys in the given
+// range. A nil Range.Start is treated as a key before all keys in the
+// DB. And a nil Range.Limit is treated as a key after all keys in
+// the DB.
+//
+// The iterator must be released after use, by calling Release method.
+//
+// Also read Iterator documentation of the leveldb/iterator package.
+func (p *DB) NewIterator(slice *util.Range) iterator.Iterator {
+	return &dbIter{p: p, slice: slice}
+}
+
+// Capacity returns keys/values buffer capacity.
+func (p *DB) Capacity() int {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return cap(p.kvData)
+}
+
+// Size returns sum of keys and values length. Note that deleted
+// key/value will not be accouted for, but it will still consume
+// the buffer, since the buffer is append only.
+func (p *DB) Size() int {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return p.kvSize
+}
+
+// Free returns keys/values free buffer before need to grow.
+func (p *DB) Free() int {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return cap(p.kvData) - len(p.kvData)
+}
+
+// Len returns the number of entries in the DB.
+func (p *DB) Len() int {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return p.n
+}
+
+// Reset resets the DB to initial empty state. Allows reuse the buffer.
+func (p *DB) Reset() {
+	p.rnd = rand.New(rand.NewSource(0xdeadbeef))
+	p.maxHeight = 1
+	p.n = 0
+	p.kvSize = 0
+	p.kvData = p.kvData[:0]
+	p.nodeData = p.nodeData[:4+tMaxHeight]
+	p.nodeData[nKV] = 0
+	p.nodeData[nKey] = 0
+	p.nodeData[nVal] = 0
+	p.nodeData[nHeight] = tMaxHeight
+	for n := 0; n < tMaxHeight; n++ {
+		p.nodeData[4+n] = 0
+		p.prevNode[n] = 0
+	}
+}
+
+// New creates a new initalized in-memory key/value DB. The capacity
+// is the initial key/value buffer capacity. The capacity is advisory,
+// not enforced.
+func New(cmp comparer.BasicComparer, capacity int) *DB {
+	p := &DB{
+		cmp:       cmp,
+		rnd:       rand.New(rand.NewSource(0xdeadbeef)),
+		maxHeight: 1,
+		kvData:    make([]byte, 0, capacity),
+		nodeData:  make([]int, 4+tMaxHeight),
+	}
+	p.nodeData[nHeight] = tMaxHeight
+	return p
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go

@@ -0,0 +1,17 @@
+package memdb
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+func TestMemdb(t *testing.T) {
+	testutil.RunDefer()
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Memdb Suite")
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package memdb
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func (p *DB) TestFindLT(key []byte) (rkey, value []byte, err error) {
+	p.mu.RLock()
+	if node := p.findLT(key); node != 0 {
+		n := p.nodeData[node]
+		m := n + p.nodeData[node+nKey]
+		rkey = p.kvData[n:m]
+		value = p.kvData[m : m+p.nodeData[node+nVal]]
+	} else {
+		err = ErrNotFound
+	}
+	p.mu.RUnlock()
+	return
+}
+
+func (p *DB) TestFindLast() (rkey, value []byte, err error) {
+	p.mu.RLock()
+	if node := p.findLast(); node != 0 {
+		n := p.nodeData[node]
+		m := n + p.nodeData[node+nKey]
+		rkey = p.kvData[n:m]
+		value = p.kvData[m : m+p.nodeData[node+nVal]]
+	} else {
+		err = ErrNotFound
+	}
+	p.mu.RUnlock()
+	return
+}
+
+func (p *DB) TestPut(key []byte, value []byte) error {
+	p.Put(key, value)
+	return nil
+}
+
+func (p *DB) TestDelete(key []byte) error {
+	p.Delete(key)
+	return nil
+}
+
+func (p *DB) TestFind(key []byte) (rkey, rvalue []byte, err error) {
+	return p.Find(key)
+}
+
+func (p *DB) TestGet(key []byte) (value []byte, err error) {
+	return p.Get(key)
+}
+
+func (p *DB) TestNewIterator(slice *util.Range) iterator.Iterator {
+	return p.NewIterator(slice)
+}
+
+var _ = testutil.Defer(func() {
+	Describe("Memdb", func() {
+		Describe("write test", func() {
+			It("should do write correctly", func() {
+				db := New(comparer.DefaultComparer, 0)
+				t := testutil.DBTesting{
+					DB:      db,
+					Deleted: testutil.KeyValue_Generate(nil, 1000, 1, 30, 5, 5).Clone(),
+					PostFn: func(t *testutil.DBTesting) {
+						Expect(db.Len()).Should(Equal(t.Present.Len()))
+						Expect(db.Size()).Should(Equal(t.Present.Size()))
+						switch t.Act {
+						case testutil.DBPut, testutil.DBOverwrite:
+							Expect(db.Contains(t.ActKey)).Should(BeTrue())
+						default:
+							Expect(db.Contains(t.ActKey)).Should(BeFalse())
+						}
+					},
+				}
+				testutil.DoDBTesting(&t)
+			})
+		})
+
+		Describe("read test", func() {
+			testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
+				// Building the DB.
+				db := New(comparer.DefaultComparer, 0)
+				kv.IterateShuffled(nil, func(i int, key, value []byte) {
+					db.Put(key, value)
+				})
+
+				if kv.Len() > 1 {
+					It("Should find correct keys with findLT", func() {
+						testutil.ShuffledIndex(nil, kv.Len()-1, 1, func(i int) {
+							key_, key, _ := kv.IndexInexact(i + 1)
+							expectedKey, expectedValue := kv.Index(i)
+
+							// Using key that exist.
+							rkey, rvalue, err := db.TestFindLT(key)
+							Expect(err).ShouldNot(HaveOccurred(), "Error for key %q -> %q", key, expectedKey)
+							Expect(rkey).Should(Equal(expectedKey), "Key")
+							Expect(rvalue).Should(Equal(expectedValue), "Value for key %q -> %q", key, expectedKey)
+
+							// Using key that doesn't exist.
+							rkey, rvalue, err = db.TestFindLT(key_)
+							Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q) -> %q", key_, key, expectedKey)
+							Expect(rkey).Should(Equal(expectedKey))
+							Expect(rvalue).Should(Equal(expectedValue), "Value for key %q (%q) -> %q", key_, key, expectedKey)
+						})
+					})
+				}
+
+				if kv.Len() > 0 {
+					It("Should find last key with findLast", func() {
+						key, value := kv.Index(kv.Len() - 1)
+						rkey, rvalue, err := db.TestFindLast()
+						Expect(err).ShouldNot(HaveOccurred())
+						Expect(rkey).Should(Equal(key))
+						Expect(rvalue).Should(Equal(value))
+					})
+				}
+
+				return db
+			})
+		})
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -0,0 +1,318 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package opt provides sets of options used by LevelDB.
+package opt
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+)
+
+const (
+	KiB = 1024
+	MiB = KiB * 1024
+	GiB = MiB * 1024
+)
+
+const (
+	DefaultBlockCacheSize       = 8 * MiB
+	DefaultBlockRestartInterval = 16
+	DefaultBlockSize            = 4 * KiB
+	DefaultCompressionType      = SnappyCompression
+	DefaultMaxOpenFiles         = 1000
+	DefaultWriteBuffer          = 4 * MiB
+)
+
+type noCache struct{}
+
+func (noCache) SetCapacity(capacity int)               {}
+func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
+func (noCache) Purge(fin cache.PurgeFin)               {}
+func (noCache) Zap(closed bool)                        {}
+
+var NoCache cache.Cache = noCache{}
+
+// Compression is the per-block compression algorithm to use.
+type Compression uint
+
+func (c Compression) String() string {
+	switch c {
+	case DefaultCompression:
+		return "default"
+	case NoCompression:
+		return "none"
+	case SnappyCompression:
+		return "snappy"
+	}
+	return "invalid"
+}
+
+const (
+	DefaultCompression Compression = iota
+	NoCompression
+	SnappyCompression
+	nCompression
+)
+
+// Strict is the DB strict level.
+type Strict uint
+
+const (
+	// If present then a corrupted or invalid chunk or block in manifest
+	// journal will cause an error istead of being dropped.
+	StrictManifest Strict = 1 << iota
+
+	// If present then a corrupted or invalid chunk or block in journal
+	// will cause an error istead of being dropped.
+	StrictJournal
+
+	// If present then journal chunk checksum will be verified.
+	StrictJournalChecksum
+
+	// If present then an invalid key/value pair will cause an error
+	// instead of being skipped.
+	StrictIterator
+
+	// If present then 'sorted table' block checksum will be verified.
+	StrictBlockChecksum
+
+	// StrictAll enables all strict flags.
+	StrictAll = StrictManifest | StrictJournal | StrictJournalChecksum | StrictIterator | StrictBlockChecksum
+
+	// DefaultStrict is the default strict flags. Specify any strict flags
+	// will override default strict flags as whole (i.e. not OR'ed).
+	DefaultStrict = StrictJournalChecksum | StrictBlockChecksum
+
+	// NoStrict disables all strict flags. Override default strict flags.
+	NoStrict = ^StrictAll
+)
+
+// Options holds the optional parameters for the DB at large.
+type Options struct {
+	// AltFilters defines one or more 'alternative filters'.
+	// 'alternative filters' will be used during reads if a filter block
+	// does not match with the 'effective filter'.
+	//
+	// The default value is nil
+	AltFilters []filter.Filter
+
+	// BlockCache provides per-block caching for LevelDB. Specify NoCache to
+	// disable block caching.
+	//
+	// By default LevelDB will create LRU-cache with capacity of 8MiB.
+	BlockCache cache.Cache
+
+	// BlockRestartInterval is the number of keys between restart points for
+	// delta encoding of keys.
+	//
+	// The default value is 16.
+	BlockRestartInterval int
+
+	// BlockSize is the minimum uncompressed size in bytes of each 'sorted table'
+	// block.
+	//
+	// The default value is 4KiB.
+	BlockSize int
+
+	// Comparer defines a total ordering over the space of []byte keys: a 'less
+	// than' relationship. The same comparison algorithm must be used for reads
+	// and writes over the lifetime of the DB.
+	//
+	// The default value uses the same ordering as bytes.Compare.
+	Comparer comparer.Comparer
+
+	// Compression defines the per-block compression to use.
+	//
+	// The default value (DefaultCompression) uses snappy compression.
+	Compression Compression
+
+	// ErrorIfExist defines whether an error should returned if the DB already
+	// exist.
+	//
+	// The default value is false.
+	ErrorIfExist bool
+
+	// ErrorIfMissing defines whether an error should returned if the DB is
+	// missing. If false then the database will be created if missing, otherwise
+	// an error will be returned.
+	//
+	// The default value is false.
+	ErrorIfMissing bool
+
+	// Filter defines an 'effective filter' to use. An 'effective filter'
+	// if defined will be used to generate per-table filter block.
+	// The filter name will be stored on disk.
+	// During reads LevelDB will try to find matching filter from
+	// 'effective filter' and 'alternative filters'.
+	//
+	// Filter can be changed after a DB has been created. It is recommended
+	// to put old filter to the 'alternative filters' to mitigate lack of
+	// filter during transition period.
+	//
+	// A filter is used to reduce disk reads when looking for a specific key.
+	//
+	// The default value is nil.
+	Filter filter.Filter
+
+	// MaxOpenFiles defines maximum number of open files to kept around
+	// (cached). This is not an hard limit, actual open files may exceed
+	// the defined value.
+	//
+	// The default value is 1000.
+	MaxOpenFiles int
+
+	// Strict defines the DB strict level.
+	Strict Strict
+
+	// WriteBuffer defines maximum size of a 'memdb' before flushed to
+	// 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk
+	// unsorted journal.
+	//
+	// LevelDB may held up to two 'memdb' at the same time.
+	//
+	// The default value is 4MiB.
+	WriteBuffer int
+}
+
+func (o *Options) GetAltFilters() []filter.Filter {
+	if o == nil {
+		return nil
+	}
+	return o.AltFilters
+}
+
+func (o *Options) GetBlockCache() cache.Cache {
+	if o == nil {
+		return nil
+	}
+	return o.BlockCache
+}
+
+func (o *Options) GetBlockRestartInterval() int {
+	if o == nil || o.BlockRestartInterval <= 0 {
+		return DefaultBlockRestartInterval
+	}
+	return o.BlockRestartInterval
+}
+
+func (o *Options) GetBlockSize() int {
+	if o == nil || o.BlockSize <= 0 {
+		return DefaultBlockSize
+	}
+	return o.BlockSize
+}
+
+func (o *Options) GetComparer() comparer.Comparer {
+	if o == nil || o.Comparer == nil {
+		return comparer.DefaultComparer
+	}
+	return o.Comparer
+}
+
+func (o *Options) GetCompression() Compression {
+	if o == nil || o.Compression <= DefaultCompression || o.Compression >= nCompression {
+		return DefaultCompressionType
+	}
+	return o.Compression
+}
+
+func (o *Options) GetErrorIfExist() bool {
+	if o == nil {
+		return false
+	}
+	return o.ErrorIfExist
+}
+
+func (o *Options) GetErrorIfMissing() bool {
+	if o == nil {
+		return false
+	}
+	return o.ErrorIfMissing
+}
+
+func (o *Options) GetFilter() filter.Filter {
+	if o == nil {
+		return nil
+	}
+	return o.Filter
+}
+
+func (o *Options) GetMaxOpenFiles() int {
+	if o == nil || o.MaxOpenFiles <= 0 {
+		return DefaultMaxOpenFiles
+	}
+	return o.MaxOpenFiles
+}
+
+func (o *Options) GetStrict(strict Strict) bool {
+	if o == nil || o.Strict == 0 {
+		return DefaultStrict&strict != 0
+	}
+	return o.Strict&strict != 0
+}
+
+func (o *Options) GetWriteBuffer() int {
+	if o == nil || o.WriteBuffer <= 0 {
+		return DefaultWriteBuffer
+	}
+	return o.WriteBuffer
+}
+
+// ReadOptions holds the optional parameters for 'read operation'. The
+// 'read operation' includes Get, Find and NewIterator.
+type ReadOptions struct {
+	// DontFillCache defines whether block reads for this 'read operation'
+	// should be cached. If false then the block will be cached. This does
+	// not affects already cached block.
+	//
+	// The default value is false.
+	DontFillCache bool
+
+	// Strict overrides global DB strict level. Only StrictIterator and
+	// StrictBlockChecksum that does have effects here.
+	Strict Strict
+}
+
+func (ro *ReadOptions) GetDontFillCache() bool {
+	if ro == nil {
+		return false
+	}
+	return ro.DontFillCache
+}
+
+func (ro *ReadOptions) GetStrict(strict Strict) bool {
+	if ro == nil {
+		return false
+	}
+	return ro.Strict&strict != 0
+}
+
+// WriteOptions holds the optional parameters for 'write operation'. The
+// 'write operation' includes Write, Put and Delete.
+type WriteOptions struct {
+	// Sync is whether to sync underlying writes from the OS buffer cache
+	// through to actual disk, if applicable. Setting Sync can result in
+	// slower writes.
+	//
+	// If false, and the machine crashes, then some recent writes may be lost.
+	// Note that if it is just the process that crashes (and the machine does
+	// not) then no writes will be lost.
+	//
+	// In other words, Sync being false has the same semantics as a write
+	// system call. Sync being true means write followed by fsync.
+	//
+	// The default value is false.
+	Sync bool
+}
+
+func (wo *WriteOptions) GetSync() bool {
+	if wo == nil {
+		return false
+	}
+	return wo.Sync
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+func (s *session) setOptions(o *opt.Options) {
+	s.o = &opt.Options{}
+	if o != nil {
+		*s.o = *o
+	}
+	// Alternative filters.
+	if filters := o.GetAltFilters(); len(filters) > 0 {
+		s.o.AltFilters = make([]filter.Filter, len(filters))
+		for i, filter := range filters {
+			s.o.AltFilters[i] = &iFilter{filter}
+		}
+	}
+	// Block cache.
+	switch o.GetBlockCache() {
+	case nil:
+		s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize)
+	case opt.NoCache:
+		s.o.BlockCache = nil
+	}
+	// Comparer.
+	s.cmp = &iComparer{o.GetComparer()}
+	s.o.Comparer = s.cmp
+	// Filter.
+	if filter := o.GetFilter(); filter != nil {
+		s.o.Filter = &iFilter{filter}
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -0,0 +1,410 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/journal"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// session represent a persistent database session.
+type session struct {
+	// Need 64-bit alignment.
+	stFileNum        uint64 // current unused file number
+	stJournalNum     uint64 // current journal file number; need external synchronization
+	stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
+	stSeq            uint64 // last mem compacted seq; need external synchronization
+	stTempFileNum    uint64
+
+	stor     storage.Storage
+	storLock util.Releaser
+	o        *opt.Options
+	cmp      *iComparer
+	tops     *tOps
+
+	manifest       *journal.Writer
+	manifestWriter storage.Writer
+	manifestFile   storage.File
+
+	stCPtrs   [kNumLevels]iKey // compact pointers; need external synchronization
+	stVersion *version         // current version
+	vmu       sync.Mutex
+}
+
+func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
+	if stor == nil {
+		return nil, os.ErrInvalid
+	}
+	storLock, err := stor.Lock()
+	if err != nil {
+		return
+	}
+	s = &session{
+		stor:     stor,
+		storLock: storLock,
+	}
+	s.setOptions(o)
+	s.tops = newTableOps(s, s.o.GetMaxOpenFiles())
+	s.setVersion(&version{s: s})
+	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
+	return
+}
+
+// Close session.
+func (s *session) close() {
+	s.tops.close()
+	if bc := s.o.GetBlockCache(); bc != nil {
+		bc.Purge(nil)
+	}
+	if s.manifest != nil {
+		s.manifest.Close()
+	}
+	if s.manifestWriter != nil {
+		s.manifestWriter.Close()
+	}
+	s.manifest = nil
+	s.manifestWriter = nil
+	s.manifestFile = nil
+	s.stVersion = nil
+}
+
+func (s *session) release() {
+	s.storLock.Release()
+}
+
+// Create a new database session; need external synchronization.
+func (s *session) create() error {
+	// create manifest
+	return s.newManifest(nil, nil)
+}
+
+// Recover a database session; need external synchronization.
+func (s *session) recover() (err error) {
+	defer func() {
+		if os.IsNotExist(err) {
+			// Don't return os.ErrNotExist if the underlying storage contains
+			// other files that belong to LevelDB. So the DB won't get trashed.
+			if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
+				err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")}
+			}
+		}
+	}()
+
+	file, err := s.stor.GetManifest()
+	if err != nil {
+		return
+	}
+
+	reader, err := file.Open()
+	if err != nil {
+		return
+	}
+	defer reader.Close()
+	strict := s.o.GetStrict(opt.StrictManifest)
+	jr := journal.NewReader(reader, dropper{s, file}, strict, true)
+
+	staging := s.version_NB().newStaging()
+	rec := &sessionRecord{}
+	for {
+		var r io.Reader
+		r, err = jr.Next()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			return
+		}
+
+		err = rec.decode(r)
+		if err == nil {
+			// save compact pointers
+			for _, rp := range rec.compactionPointers {
+				s.stCPtrs[rp.level] = iKey(rp.key)
+			}
+			// commit record to version staging
+			staging.commit(rec)
+		} else if strict {
+			return ErrCorrupted{Type: CorruptedManifest, Err: err}
+		} else {
+			s.logf("manifest error: %v (skipped)", err)
+		}
+		rec.resetCompactionPointers()
+		rec.resetAddedTables()
+		rec.resetDeletedTables()
+	}
+
+	switch {
+	case !rec.has(recComparer):
+		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")}
+	case rec.comparer != s.cmp.cmp.Name():
+		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.cmp.cmp.Name() + "', " + "got '" + rec.comparer + "'")}
+	case !rec.has(recNextNum):
+		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")}
+	case !rec.has(recJournalNum):
+		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")}
+	case !rec.has(recSeq):
+		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")}
+	}
+
+	s.manifestFile = file
+	s.setVersion(staging.finish())
+	s.setFileNum(rec.nextNum)
+	s.recordCommited(rec)
+	return nil
+}
+
+// Commit session; need external synchronization.
+func (s *session) commit(r *sessionRecord) (err error) {
+	// spawn new version based on current version
+	nv := s.version_NB().spawn(r)
+
+	if s.manifest == nil {
+		// manifest journal writer not yet created, create one
+		err = s.newManifest(r, nv)
+	} else {
+		err = s.flushManifest(r)
+	}
+
+	// finally, apply new version if no error rise
+	if err == nil {
+		s.setVersion(nv)
+	}
+
+	return
+}
+
+// Pick a compaction based on current state; need external synchronization.
+func (s *session) pickCompaction() *compaction {
+	icmp := s.cmp
+	ucmp := icmp.cmp
+
+	v := s.version_NB()
+
+	var level int
+	var t0 tFiles
+	if v.cScore >= 1 {
+		level = v.cLevel
+		cp := s.stCPtrs[level]
+		tt := v.tables[level]
+		for _, t := range tt {
+			if cp == nil || icmp.Compare(t.max, cp) > 0 {
+				t0 = append(t0, t)
+				break
+			}
+		}
+		if len(t0) == 0 {
+			t0 = append(t0, tt[0])
+		}
+	} else {
+		if p := atomic.LoadPointer(&v.cSeek); p != nil {
+			ts := (*tSet)(p)
+			level = ts.level
+			t0 = append(t0, ts.table)
+		} else {
+			return nil
+		}
+	}
+
+	c := &compaction{s: s, version: v, level: level}
+	if level == 0 {
+		min, max := t0.getRange(icmp)
+		t0 = nil
+		v.tables[0].getOverlaps(min.ukey(), max.ukey(), &t0, false, ucmp)
+	}
+
+	c.tables[0] = t0
+	c.expand()
+	return c
+}
+
+// Create compaction from given level and range; need external synchronization.
+func (s *session) getCompactionRange(level int, min, max []byte) *compaction {
+	v := s.version_NB()
+
+	var t0 tFiles
+	v.tables[level].getOverlaps(min, max, &t0, level != 0, s.cmp.cmp)
+	if len(t0) == 0 {
+		return nil
+	}
+
+	// Avoid compacting too much in one shot in case the range is large.
+	// But we cannot do this for level-0 since level-0 files can overlap
+	// and we must not pick one file and drop another older file if the
+	// two files overlap.
+	if level > 0 {
+		limit := uint64(kMaxTableSize)
+		total := uint64(0)
+		for i, t := range t0 {
+			total += t.size
+			if total >= limit {
+				s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
+				t0 = t0[:i+1]
+				break
+			}
+		}
+	}
+
+	c := &compaction{s: s, version: v, level: level}
+	c.tables[0] = t0
+	c.expand()
+	return c
+}
+
+// compaction represent a compaction state
+type compaction struct {
+	s       *session
+	version *version
+
+	level  int
+	tables [2]tFiles
+
+	gp              tFiles
+	gpidx           int
+	seenKey         bool
+	overlappedBytes uint64
+	min, max        iKey
+
+	tPtrs [kNumLevels]int
+}
+
+// Expand compacted tables; need external synchronization.
+func (c *compaction) expand() {
+	s := c.s
+	v := c.version
+	icmp := s.cmp
+	ucmp := icmp.cmp
+
+	level := c.level
+	vt0, vt1 := v.tables[level], v.tables[level+1]
+
+	t0, t1 := c.tables[0], c.tables[1]
+	min, max := t0.getRange(icmp)
+	vt1.getOverlaps(min.ukey(), max.ukey(), &t1, true, ucmp)
+
+	// Get entire range covered by compaction
+	amin, amax := append(t0, t1...).getRange(icmp)
+
+	// See if we can grow the number of inputs in "level" without
+	// changing the number of "level+1" files we pick up.
+	if len(t1) > 0 {
+		var exp0 tFiles
+		vt0.getOverlaps(amin.ukey(), amax.ukey(), &exp0, level != 0, ucmp)
+		if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes {
+			var exp1 tFiles
+			xmin, xmax := exp0.getRange(icmp)
+			vt1.getOverlaps(xmin.ukey(), xmax.ukey(), &exp1, true, ucmp)
+			if len(exp1) == len(t1) {
+				s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
+					level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
+					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
+				min, max = xmin, xmax
+				t0, t1 = exp0, exp1
+				amin, amax = append(t0, t1...).getRange(icmp)
+			}
+		}
+	}
+
+	// Compute the set of grandparent files that overlap this compaction
+	// (parent == level+1; grandparent == level+2)
+	if level+2 < kNumLevels {
+		v.tables[level+2].getOverlaps(amin.ukey(), amax.ukey(), &c.gp, true, ucmp)
+	}
+
+	c.tables[0], c.tables[1] = t0, t1
+	c.min, c.max = min, max
+}
+
+// Check whether compaction is trivial.
+func (c *compaction) trivial() bool {
+	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes
+}
+
+func (c *compaction) isBaseLevelForKey(key []byte) bool {
+	s := c.s
+	v := c.version
+	ucmp := s.cmp.cmp
+	for level, tt := range v.tables[c.level+2:] {
+		for c.tPtrs[level] < len(tt) {
+			t := tt[c.tPtrs[level]]
+			if ucmp.Compare(key, t.max.ukey()) <= 0 {
+				// We've advanced far enough
+				if ucmp.Compare(key, t.min.ukey()) >= 0 {
+					// Key falls in this file's range, so definitely not base level
+					return false
+				}
+				break
+			}
+			c.tPtrs[level]++
+		}
+	}
+	return true
+}
+
+func (c *compaction) shouldStopBefore(key iKey) bool {
+	icmp := c.s.cmp
+	for ; c.gpidx < len(c.gp); c.gpidx++ {
+		gp := c.gp[c.gpidx]
+		if icmp.Compare(key, gp.max) <= 0 {
+			break
+		}
+		if c.seenKey {
+			c.overlappedBytes += gp.size
+		}
+	}
+	c.seenKey = true
+
+	if c.overlappedBytes > kMaxGrandParentOverlapBytes {
+		// Too much overlap for current output; start new output
+		c.overlappedBytes = 0
+		return true
+	}
+	return false
+}
+
+func (c *compaction) newIterator() iterator.Iterator {
+	s := c.s
+	icmp := s.cmp
+
+	level := c.level
+	icap := 2
+	if c.level == 0 {
+		icap = len(c.tables[0]) + 1
+	}
+	its := make([]iterator.Iterator, 0, icap)
+
+	ro := &opt.ReadOptions{
+		DontFillCache: true,
+	}
+	strict := s.o.GetStrict(opt.StrictIterator)
+
+	for i, tt := range c.tables {
+		if len(tt) == 0 {
+			continue
+		}
+
+		if level+i == 0 {
+			for _, t := range tt {
+				its = append(its, s.tops.newIterator(t, nil, ro))
+			}
+		} else {
+			it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, icmp, nil, ro), strict, true)
+			its = append(its, it)
+		}
+	}
+
+	return iterator.NewMergedIterator(its, icmp, true)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go

@@ -0,0 +1,308 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bufio"
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var errCorruptManifest = errors.New("leveldb: corrupt manifest")
+
+type byteReader interface {
+	io.Reader
+	io.ByteReader
+}
+
+// These numbers are written to disk and should not be changed.
+const (
+	recComparer          = 1
+	recJournalNum        = 2
+	recNextNum           = 3
+	recSeq               = 4
+	recCompactionPointer = 5
+	recDeletedTable      = 6
+	recNewTable          = 7
+	// 8 was used for large value refs
+	recPrevJournalNum = 9
+)
+
+type cpRecord struct {
+	level int
+	key   iKey
+}
+
+type ntRecord struct {
+	level int
+	num   uint64
+	size  uint64
+	min   iKey
+	max   iKey
+}
+
+func (r ntRecord) makeFile(s *session) *tFile {
+	return newTFile(s.getTableFile(r.num), r.size, r.min, r.max)
+}
+
+type dtRecord struct {
+	level int
+	num   uint64
+}
+
+type sessionRecord struct {
+	hasRec             int
+	comparer           string
+	journalNum         uint64
+	prevJournalNum     uint64
+	nextNum            uint64
+	seq                uint64
+	compactionPointers []cpRecord
+	addedTables        []ntRecord
+	deletedTables      []dtRecord
+	scratch            [binary.MaxVarintLen64]byte
+	err                error
+}
+
+func (p *sessionRecord) has(rec int) bool {
+	return p.hasRec&(1<<uint(rec)) != 0
+}
+
+func (p *sessionRecord) setComparer(name string) {
+	p.hasRec |= 1 << recComparer
+	p.comparer = name
+}
+
+func (p *sessionRecord) setJournalNum(num uint64) {
+	p.hasRec |= 1 << recJournalNum
+	p.journalNum = num
+}
+
+func (p *sessionRecord) setPrevJournalNum(num uint64) {
+	p.hasRec |= 1 << recPrevJournalNum
+	p.prevJournalNum = num
+}
+
+func (p *sessionRecord) setNextNum(num uint64) {
+	p.hasRec |= 1 << recNextNum
+	p.nextNum = num
+}
+
+func (p *sessionRecord) setSeq(seq uint64) {
+	p.hasRec |= 1 << recSeq
+	p.seq = seq
+}
+
+func (p *sessionRecord) addCompactionPointer(level int, key iKey) {
+	p.hasRec |= 1 << recCompactionPointer
+	p.compactionPointers = append(p.compactionPointers, cpRecord{level, key})
+}
+
+func (p *sessionRecord) resetCompactionPointers() {
+	p.hasRec &= ^(1 << recCompactionPointer)
+	p.compactionPointers = p.compactionPointers[:0]
+}
+
+func (p *sessionRecord) addTable(level int, num, size uint64, min, max iKey) {
+	p.hasRec |= 1 << recNewTable
+	p.addedTables = append(p.addedTables, ntRecord{level, num, size, min, max})
+}
+
+func (p *sessionRecord) addTableFile(level int, t *tFile) {
+	p.addTable(level, t.file.Num(), t.size, t.min, t.max)
+}
+
+func (p *sessionRecord) resetAddedTables() {
+	p.hasRec &= ^(1 << recNewTable)
+	p.addedTables = p.addedTables[:0]
+}
+
+func (p *sessionRecord) deleteTable(level int, num uint64) {
+	p.hasRec |= 1 << recDeletedTable
+	p.deletedTables = append(p.deletedTables, dtRecord{level, num})
+}
+
+func (p *sessionRecord) resetDeletedTables() {
+	p.hasRec &= ^(1 << recDeletedTable)
+	p.deletedTables = p.deletedTables[:0]
+}
+
+func (p *sessionRecord) putUvarint(w io.Writer, x uint64) {
+	if p.err != nil {
+		return
+	}
+	n := binary.PutUvarint(p.scratch[:], x)
+	_, p.err = w.Write(p.scratch[:n])
+}
+
+func (p *sessionRecord) putBytes(w io.Writer, x []byte) {
+	if p.err != nil {
+		return
+	}
+	p.putUvarint(w, uint64(len(x)))
+	if p.err != nil {
+		return
+	}
+	_, p.err = w.Write(x)
+}
+
+func (p *sessionRecord) encode(w io.Writer) error {
+	p.err = nil
+	if p.has(recComparer) {
+		p.putUvarint(w, recComparer)
+		p.putBytes(w, []byte(p.comparer))
+	}
+	if p.has(recJournalNum) {
+		p.putUvarint(w, recJournalNum)
+		p.putUvarint(w, p.journalNum)
+	}
+	if p.has(recNextNum) {
+		p.putUvarint(w, recNextNum)
+		p.putUvarint(w, p.nextNum)
+	}
+	if p.has(recSeq) {
+		p.putUvarint(w, recSeq)
+		p.putUvarint(w, p.seq)
+	}
+	for _, cp := range p.compactionPointers {
+		p.putUvarint(w, recCompactionPointer)
+		p.putUvarint(w, uint64(cp.level))
+		p.putBytes(w, cp.key)
+	}
+	for _, t := range p.deletedTables {
+		p.putUvarint(w, recDeletedTable)
+		p.putUvarint(w, uint64(t.level))
+		p.putUvarint(w, t.num)
+	}
+	for _, t := range p.addedTables {
+		p.putUvarint(w, recNewTable)
+		p.putUvarint(w, uint64(t.level))
+		p.putUvarint(w, t.num)
+		p.putUvarint(w, t.size)
+		p.putBytes(w, t.min)
+		p.putBytes(w, t.max)
+	}
+	return p.err
+}
+
+func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
+	if p.err != nil {
+		return 0
+	}
+	x, err := binary.ReadUvarint(r)
+	if err != nil {
+		if err == io.EOF {
+			p.err = errCorruptManifest
+		} else {
+			p.err = err
+		}
+		return 0
+	}
+	return x
+}
+
+func (p *sessionRecord) readBytes(r byteReader) []byte {
+	if p.err != nil {
+		return nil
+	}
+	n := p.readUvarint(r)
+	if p.err != nil {
+		return nil
+	}
+	x := make([]byte, n)
+	_, p.err = io.ReadFull(r, x)
+	if p.err != nil {
+		if p.err == io.EOF {
+			p.err = errCorruptManifest
+		}
+		return nil
+	}
+	return x
+}
+
+func (p *sessionRecord) readLevel(r io.ByteReader) int {
+	if p.err != nil {
+		return 0
+	}
+	x := p.readUvarint(r)
+	if p.err != nil {
+		return 0
+	}
+	if x >= kNumLevels {
+		p.err = errCorruptManifest
+		return 0
+	}
+	return int(x)
+}
+
+func (p *sessionRecord) decode(r io.Reader) error {
+	br, ok := r.(byteReader)
+	if !ok {
+		br = bufio.NewReader(r)
+	}
+	p.err = nil
+	for p.err == nil {
+		rec, err := binary.ReadUvarint(br)
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return err
+		}
+		switch rec {
+		case recComparer:
+			x := p.readBytes(br)
+			if p.err == nil {
+				p.setComparer(string(x))
+			}
+		case recJournalNum:
+			x := p.readUvarint(br)
+			if p.err == nil {
+				p.setJournalNum(x)
+			}
+		case recPrevJournalNum:
+			x := p.readUvarint(br)
+			if p.err == nil {
+				p.setPrevJournalNum(x)
+			}
+		case recNextNum:
+			x := p.readUvarint(br)
+			if p.err == nil {
+				p.setNextNum(x)
+			}
+		case recSeq:
+			x := p.readUvarint(br)
+			if p.err == nil {
+				p.setSeq(x)
+			}
+		case recCompactionPointer:
+			level := p.readLevel(br)
+			key := p.readBytes(br)
+			if p.err == nil {
+				p.addCompactionPointer(level, iKey(key))
+			}
+		case recNewTable:
+			level := p.readLevel(br)
+			num := p.readUvarint(br)
+			size := p.readUvarint(br)
+			min := p.readBytes(br)
+			max := p.readBytes(br)
+			if p.err == nil {
+				p.addTable(level, num, size, min, max)
+			}
+		case recDeletedTable:
+			level := p.readLevel(br)
+			num := p.readUvarint(br)
+			if p.err == nil {
+				p.deleteTable(level, num)
+			}
+		}
+	}
+
+	return p.err
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"bytes"
+	"testing"
+)
+
+func decodeEncode(v *sessionRecord) (res bool, err error) {
+	b := new(bytes.Buffer)
+	err = v.encode(b)
+	if err != nil {
+		return
+	}
+	v2 := new(sessionRecord)
+	err = v.decode(b)
+	if err != nil {
+		return
+	}
+	b2 := new(bytes.Buffer)
+	err = v2.encode(b2)
+	if err != nil {
+		return
+	}
+	return bytes.Equal(b.Bytes(), b2.Bytes()), nil
+}
+
+func TestSessionRecord_EncodeDecode(t *testing.T) {
+	big := uint64(1) << 50
+	v := new(sessionRecord)
+	i := uint64(0)
+	test := func() {
+		res, err := decodeEncode(v)
+		if err != nil {
+			t.Fatalf("error when testing encode/decode sessionRecord: %v", err)
+		}
+		if !res {
+			t.Error("encode/decode test failed at iteration:", i)
+		}
+	}
+
+	for ; i < 4; i++ {
+		test()
+		v.addTable(3, big+300+i, big+400+i,
+			newIKey([]byte("foo"), big+500+1, tVal),
+			newIKey([]byte("zoo"), big+600+1, tDel))
+		v.deleteTable(4, big+700+i)
+		v.addCompactionPointer(int(i), newIKey([]byte("x"), big+900+1, tVal))
+	}
+
+	v.setComparer("foo")
+	v.setJournalNum(big + 100)
+	v.setPrevJournalNum(big + 99)
+	v.setNextNum(big + 200)
+	v.setSeq(big + 1000)
+	test()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -0,0 +1,253 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/journal"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+// logging
+
+type dropper struct {
+	s    *session
+	file storage.File
+}
+
+func (d dropper) Drop(err error) {
+	if e, ok := err.(journal.DroppedError); ok {
+		d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
+	} else {
+		d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
+	}
+}
+
+func (s *session) log(v ...interface{}) {
+	s.stor.Log(fmt.Sprint(v...))
+}
+
+func (s *session) logf(format string, v ...interface{}) {
+	s.stor.Log(fmt.Sprintf(format, v...))
+}
+
+// file utils
+
+func (s *session) getJournalFile(num uint64) storage.File {
+	return s.stor.GetFile(num, storage.TypeJournal)
+}
+
+func (s *session) getTableFile(num uint64) storage.File {
+	return s.stor.GetFile(num, storage.TypeTable)
+}
+
+func (s *session) getFiles(t storage.FileType) ([]storage.File, error) {
+	return s.stor.GetFiles(t)
+}
+
+func (s *session) newTemp() storage.File {
+	num := atomic.AddUint64(&s.stTempFileNum, 1) - 1
+	return s.stor.GetFile(num, storage.TypeTemp)
+}
+
+// session state
+
+// Get current version.
+func (s *session) version() *version {
+	s.vmu.Lock()
+	defer s.vmu.Unlock()
+	s.stVersion.ref++
+	return s.stVersion
+}
+
+// Get current version; no barrier.
+func (s *session) version_NB() *version {
+	return s.stVersion
+}
+
+// Set current version to v.
+func (s *session) setVersion(v *version) {
+	s.vmu.Lock()
+	v.ref = 1
+	if old := s.stVersion; old != nil {
+		v.ref++
+		old.next = v
+		old.release_NB()
+	}
+	s.stVersion = v
+	s.vmu.Unlock()
+}
+
+// Get current unused file number.
+func (s *session) fileNum() uint64 {
+	return atomic.LoadUint64(&s.stFileNum)
+}
+
+// Get current unused file number to num.
+func (s *session) setFileNum(num uint64) {
+	atomic.StoreUint64(&s.stFileNum, num)
+}
+
+// Mark file number as used.
+func (s *session) markFileNum(num uint64) {
+	num += 1
+	for {
+		old, x := s.stFileNum, num
+		if old > x {
+			x = old
+		}
+		if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
+			break
+		}
+	}
+}
+
+// Allocate a file number.
+func (s *session) allocFileNum() (num uint64) {
+	return atomic.AddUint64(&s.stFileNum, 1) - 1
+}
+
+// Reuse given file number.
+func (s *session) reuseFileNum(num uint64) {
+	for {
+		old, x := s.stFileNum, num
+		if old != x+1 {
+			x = old
+		}
+		if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
+			break
+		}
+	}
+}
+
+// manifest related utils
+
+// Fill given session record obj with current states; need external
+// synchronization.
+func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
+	r.setNextNum(s.fileNum())
+
+	if snapshot {
+		if !r.has(recJournalNum) {
+			r.setJournalNum(s.stJournalNum)
+		}
+
+		if !r.has(recSeq) {
+			r.setSeq(s.stSeq)
+		}
+
+		for level, ik := range s.stCPtrs {
+			if ik != nil {
+				r.addCompactionPointer(level, ik)
+			}
+		}
+
+		r.setComparer(s.cmp.cmp.Name())
+	}
+}
+
+// Mark if record has been commited, this will update session state;
+// need external synchronization.
+func (s *session) recordCommited(r *sessionRecord) {
+	if r.has(recJournalNum) {
+		s.stJournalNum = r.journalNum
+	}
+
+	if r.has(recPrevJournalNum) {
+		s.stPrevJournalNum = r.prevJournalNum
+	}
+
+	if r.has(recSeq) {
+		s.stSeq = r.seq
+	}
+
+	for _, p := range r.compactionPointers {
+		s.stCPtrs[p.level] = iKey(p.key)
+	}
+}
+
+// Create a new manifest file; need external synchronization.
+func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
+	num := s.allocFileNum()
+	file := s.stor.GetFile(num, storage.TypeManifest)
+	writer, err := file.Create()
+	if err != nil {
+		return
+	}
+	jw := journal.NewWriter(writer)
+
+	if v == nil {
+		v = s.version_NB()
+	}
+	if rec == nil {
+		rec = new(sessionRecord)
+	}
+	s.fillRecord(rec, true)
+	v.fillRecord(rec)
+
+	defer func() {
+		if err == nil {
+			s.recordCommited(rec)
+			if s.manifest != nil {
+				s.manifest.Close()
+			}
+			if s.manifestWriter != nil {
+				s.manifestWriter.Close()
+			}
+			if s.manifestFile != nil {
+				s.manifestFile.Remove()
+			}
+			s.manifestFile = file
+			s.manifestWriter = writer
+			s.manifest = jw
+		} else {
+			writer.Close()
+			file.Remove()
+			s.reuseFileNum(num)
+		}
+	}()
+
+	w, err := jw.Next()
+	if err != nil {
+		return
+	}
+	err = rec.encode(w)
+	if err != nil {
+		return
+	}
+	err = jw.Flush()
+	if err != nil {
+		return
+	}
+	err = s.stor.SetManifest(file)
+	return
+}
+
+// Flush record to disk.
+func (s *session) flushManifest(rec *sessionRecord) (err error) {
+	s.fillRecord(rec, false)
+	w, err := s.manifest.Next()
+	if err != nil {
+		return
+	}
+	err = rec.encode(w)
+	if err != nil {
+		return
+	}
+	err = s.manifest.Flush()
+	if err != nil {
+		return
+	}
+	err = s.manifestWriter.Sync()
+	if err != nil {
+		return
+	}
+	s.recordCommited(rec)
+	return
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -0,0 +1,536 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reservefs.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var errFileOpen = errors.New("leveldb/storage: file still open")
+
+type fileLock interface {
+	release() error
+}
+
+type fileStorageLock struct {
+	fs *fileStorage
+}
+
+func (lock *fileStorageLock) Release() {
+	fs := lock.fs
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.slock == lock {
+		fs.slock = nil
+	}
+	return
+}
+
+// fileStorage is a file-system backed storage.
+type fileStorage struct {
+	path string
+
+	mu    sync.Mutex
+	flock fileLock
+	slock *fileStorageLock
+	logw  *os.File
+	buf   []byte
+	// Opened file counter; if open < 0 means closed.
+	open int
+	day  int
+}
+
+// OpenFile returns a new filesytem-backed storage implementation with the given
+// path. This also hold a file lock, so any subsequent attempt to open the same
+// path will fail.
+//
+// The storage must be closed after use, by calling Close method.
+func OpenFile(path string) (Storage, error) {
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, err
+	}
+
+	flock, err := newFileLock(filepath.Join(path, "LOCK"))
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			flock.release()
+		}
+	}()
+
+	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
+	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	fs := &fileStorage{path: path, flock: flock, logw: logw}
+	runtime.SetFinalizer(fs, (*fileStorage).Close)
+	return fs, nil
+}
+
+func (fs *fileStorage) Lock() (util.Releaser, error) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return nil, ErrClosed
+	}
+	if fs.slock != nil {
+		return nil, ErrLocked
+	}
+	fs.slock = &fileStorageLock{fs: fs}
+	return fs.slock, nil
+}
+
+func itoa(buf []byte, i int, wid int) []byte {
+	var u uint = uint(i)
+	if u == 0 && wid <= 1 {
+		return append(buf, '0')
+	}
+
+	// Assemble decimal in reverse order.
+	var b [32]byte
+	bp := len(b)
+	for ; u > 0 || wid > 0; u /= 10 {
+		bp--
+		wid--
+		b[bp] = byte(u%10) + '0'
+	}
+	return append(buf, b[bp:]...)
+}
+
+func (fs *fileStorage) printDay(t time.Time) {
+	if fs.day == t.Day() {
+		return
+	}
+	fs.day = t.Day()
+	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
+}
+
+func (fs *fileStorage) doLog(t time.Time, str string) {
+	fs.printDay(t)
+	hour, min, sec := t.Clock()
+	msec := t.Nanosecond() / 1e3
+	// time
+	fs.buf = itoa(fs.buf[:0], hour, 2)
+	fs.buf = append(fs.buf, ':')
+	fs.buf = itoa(fs.buf, min, 2)
+	fs.buf = append(fs.buf, ':')
+	fs.buf = itoa(fs.buf, sec, 2)
+	fs.buf = append(fs.buf, '.')
+	fs.buf = itoa(fs.buf, msec, 6)
+	fs.buf = append(fs.buf, ' ')
+	// write
+	fs.buf = append(fs.buf, []byte(str)...)
+	fs.buf = append(fs.buf, '\n')
+	fs.logw.Write(fs.buf)
+}
+
+func (fs *fileStorage) Log(str string) {
+	t := time.Now()
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return
+	}
+	fs.doLog(t, str)
+}
+
+func (fs *fileStorage) log(str string) {
+	fs.doLog(time.Now(), str)
+}
+
+func (fs *fileStorage) GetFile(num uint64, t FileType) File {
+	return &file{fs: fs, num: num, t: t}
+}
+
+func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return nil, ErrClosed
+	}
+	dir, err := os.Open(fs.path)
+	if err != nil {
+		return
+	}
+	fnn, err := dir.Readdirnames(0)
+	// Close the dir first before checking for Readdirnames error.
+	if err := dir.Close(); err != nil {
+		fs.log(fmt.Sprintf("close dir: %v", err))
+	}
+	if err != nil {
+		return
+	}
+	f := &file{fs: fs}
+	for _, fn := range fnn {
+		if f.parse(fn) && (f.t&t) != 0 {
+			ff = append(ff, f)
+			f = &file{fs: fs}
+		}
+	}
+	return
+}
+
+func (fs *fileStorage) GetManifest() (f File, err error) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return nil, ErrClosed
+	}
+	dir, err := os.Open(fs.path)
+	if err != nil {
+		return
+	}
+	fnn, err := dir.Readdirnames(0)
+	// Close the dir first before checking for Readdirnames error.
+	if err := dir.Close(); err != nil {
+		fs.log(fmt.Sprintf("close dir: %v", err))
+	}
+	if err != nil {
+		return
+	}
+	// Find latest CURRENT file.
+	var rem []string
+	var pend bool
+	var cerr error
+	for _, fn := range fnn {
+		if strings.HasPrefix(fn, "CURRENT") {
+			pend1 := len(fn) > 7
+			// Make sure it is valid name for a CURRENT file, otherwise skip it.
+			if pend1 {
+				if fn[7] != '.' || len(fn) < 9 {
+					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
+					continue
+				}
+				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
+					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
+					continue
+				}
+			}
+			path := filepath.Join(fs.path, fn)
+			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
+			if e1 != nil {
+				return nil, e1
+			}
+			b, e1 := ioutil.ReadAll(r)
+			if e1 != nil {
+				r.Close()
+				return nil, e1
+			}
+			f1 := &file{fs: fs}
+			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
+				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
+				if pend1 {
+					rem = append(rem, fn)
+				}
+				if !pend1 || cerr == nil {
+					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
+				}
+			} else if f != nil && f1.Num() < f.Num() {
+				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
+				if pend1 {
+					rem = append(rem, fn)
+				}
+			} else {
+				f = f1
+				pend = pend1
+			}
+			if err := r.Close(); err != nil {
+				fs.log(fmt.Sprintf("close %s: %v", fn, err))
+			}
+		}
+	}
+	// Don't remove any files if there is no valid CURRENT file.
+	if f == nil {
+		if cerr != nil {
+			err = cerr
+		} else {
+			err = os.ErrNotExist
+		}
+		return
+	}
+	// Rename pending CURRENT file to an effective CURRENT.
+	if pend {
+		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
+		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
+			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", f.Num(), err))
+		}
+	}
+	// Remove obsolete or incomplete pending CURRENT files.
+	for _, fn := range rem {
+		path := filepath.Join(fs.path, fn)
+		if err := os.Remove(path); err != nil {
+			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
+		}
+	}
+	return
+}
+
+func (fs *fileStorage) SetManifest(f File) (err error) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return ErrClosed
+	}
+	f2, ok := f.(*file)
+	if !ok || f2.t != TypeManifest {
+		return ErrInvalidFile
+	}
+	defer func() {
+		if err != nil {
+			fs.log(fmt.Sprintf("CURRENT: %v", err))
+		}
+	}()
+	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
+	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintln(w, f2.name())
+	// Close the file first.
+	if err := w.Close(); err != nil {
+		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
+	}
+	if err != nil {
+		return err
+	}
+	return rename(path, filepath.Join(fs.path, "CURRENT"))
+}
+
+func (fs *fileStorage) Close() error {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	if fs.open < 0 {
+		return ErrClosed
+	}
+	// Clear the finalizer.
+	runtime.SetFinalizer(fs, nil)
+
+	if fs.open > 0 {
+		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
+		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
+	}
+	fs.open = -1
+	e1 := fs.logw.Close()
+	err := fs.flock.release()
+	if err == nil {
+		err = e1
+	}
+	return err
+}
+
+type fileWrap struct {
+	*os.File
+	f *file
+}
+
+func (fw fileWrap) Sync() error {
+	if fw.f.Type() == TypeManifest {
+		// Also sync parent directory if file type is manifest.
+		// See: https://code.google.com/p/leveldb/issues/detail?id=190.
+		f, err := os.Open(fw.f.fs.path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		if err := f.Sync(); err != nil {
+			return err
+		}
+	}
+	return fw.File.Sync()
+}
+
+func (fw fileWrap) Close() error {
+	f := fw.f
+	f.fs.mu.Lock()
+	defer f.fs.mu.Unlock()
+	if !f.open {
+		return ErrClosed
+	}
+	f.open = false
+	f.fs.open--
+	err := fw.File.Close()
+	if err != nil {
+		f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
+	}
+	return err
+}
+
+type file struct {
+	fs   *fileStorage
+	num  uint64
+	t    FileType
+	open bool
+}
+
+func (f *file) Open() (Reader, error) {
+	f.fs.mu.Lock()
+	defer f.fs.mu.Unlock()
+	if f.fs.open < 0 {
+		return nil, ErrClosed
+	}
+	if f.open {
+		return nil, errFileOpen
+	}
+	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
+	if err != nil {
+		if f.hasOldName() && os.IsNotExist(err) {
+			of, err = os.OpenFile(f.oldPath(), os.O_RDONLY, 0)
+			if err == nil {
+				goto ok
+			}
+		}
+		return nil, err
+	}
+ok:
+	f.open = true
+	f.fs.open++
+	return fileWrap{of, f}, nil
+}
+
+func (f *file) Create() (Writer, error) {
+	f.fs.mu.Lock()
+	defer f.fs.mu.Unlock()
+	if f.fs.open < 0 {
+		return nil, ErrClosed
+	}
+	if f.open {
+		return nil, errFileOpen
+	}
+	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return nil, err
+	}
+	f.open = true
+	f.fs.open++
+	return fileWrap{of, f}, nil
+}
+
+func (f *file) Replace(newfile File) error {
+	f.fs.mu.Lock()
+	defer f.fs.mu.Unlock()
+	if f.fs.open < 0 {
+		return ErrClosed
+	}
+	newfile2, ok := newfile.(*file)
+	if !ok {
+		return ErrInvalidFile
+	}
+	if f.open || newfile2.open {
+		return errFileOpen
+	}
+	return rename(newfile2.path(), f.path())
+}
+
+func (f *file) Type() FileType {
+	return f.t
+}
+
+func (f *file) Num() uint64 {
+	return f.num
+}
+
+func (f *file) Remove() error {
+	f.fs.mu.Lock()
+	defer f.fs.mu.Unlock()
+	if f.fs.open < 0 {
+		return ErrClosed
+	}
+	if f.open {
+		return errFileOpen
+	}
+	err := os.Remove(f.path())
+	if err != nil {
+		f.fs.log(fmt.Sprintf("remove %s.%d: %v", f.Type(), f.Num(), err))
+	}
+	// Also try remove file with old name, just in case.
+	if f.hasOldName() {
+		if e1 := os.Remove(f.oldPath()); !os.IsNotExist(e1) {
+			f.fs.log(fmt.Sprintf("remove %s.%d: %v (old name)", f.Type(), f.Num(), err))
+			err = e1
+		}
+	}
+	return err
+}
+
+func (f *file) hasOldName() bool {
+	return f.t == TypeTable
+}
+
+func (f *file) oldName() string {
+	switch f.t {
+	case TypeTable:
+		return fmt.Sprintf("%06d.sst", f.num)
+	}
+	return f.name()
+}
+
+func (f *file) oldPath() string {
+	return filepath.Join(f.fs.path, f.oldName())
+}
+
+func (f *file) name() string {
+	switch f.t {
+	case TypeManifest:
+		return fmt.Sprintf("MANIFEST-%06d", f.num)
+	case TypeJournal:
+		return fmt.Sprintf("%06d.log", f.num)
+	case TypeTable:
+		return fmt.Sprintf("%06d.ldb", f.num)
+	case TypeTemp:
+		return fmt.Sprintf("%06d.tmp", f.num)
+	default:
+		panic("invalid file type")
+	}
+}
+
+func (f *file) path() string {
+	return filepath.Join(f.fs.path, f.name())
+}
+
+func (f *file) parse(name string) bool {
+	var num uint64
+	var tail string
+	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
+	if err == nil {
+		switch tail {
+		case "log":
+			f.t = TypeJournal
+		case "ldb", "sst":
+			f.t = TypeTable
+		case "tmp":
+			f.t = TypeTemp
+		default:
+			return false
+		}
+		f.num = num
+		return true
+	}
+	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
+	if n == 1 {
+		f.t = TypeManifest
+		f.num = num
+		return true
+	}
+
+	return false
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"os"
+	"path/filepath"
+)
+
+type plan9FileLock struct {
+	f *os.File
+}
+
+func (fl *plan9FileLock) release() error {
+	return fl.f.Close()
+}
+
+func newFileLock(path string) (fl fileLock, err error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
+	if err != nil {
+		return
+	}
+	fl = &plan9FileLock{f: f}
+	return
+}
+
+func rename(oldpath, newpath string) error {
+	if _, err := os.Stat(newpath); err == nil {
+		if err := os.Remove(newpath); err != nil {
+			return err
+		}
+	}
+
+	_, fname := filepath.Split(newpath)
+	return os.Rename(oldpath, fname)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var cases = []struct {
+	oldName []string
+	name    string
+	ftype   FileType
+	num     uint64
+}{
+	{nil, "000100.log", TypeJournal, 100},
+	{nil, "000000.log", TypeJournal, 0},
+	{[]string{"000000.sst"}, "000000.ldb", TypeTable, 0},
+	{nil, "MANIFEST-000002", TypeManifest, 2},
+	{nil, "MANIFEST-000007", TypeManifest, 7},
+	{nil, "18446744073709551615.log", TypeJournal, 18446744073709551615},
+	{nil, "000100.tmp", TypeTemp, 100},
+}
+
+var invalidCases = []string{
+	"",
+	"foo",
+	"foo-dx-100.log",
+	".log",
+	"",
+	"manifest",
+	"CURREN",
+	"CURRENTX",
+	"MANIFES",
+	"MANIFEST",
+	"MANIFEST-",
+	"XMANIFEST-3",
+	"MANIFEST-3x",
+	"LOC",
+	"LOCKx",
+	"LO",
+	"LOGx",
+	"18446744073709551616.log",
+	"184467440737095516150.log",
+	"100",
+	"100.",
+	"100.lop",
+}
+
+func TestFileStorage_CreateFileName(t *testing.T) {
+	for _, c := range cases {
+		f := &file{num: c.num, t: c.ftype}
+		if f.name() != c.name {
+			t.Errorf("invalid filename got '%s', want '%s'", f.name(), c.name)
+		}
+	}
+}
+
+func TestFileStorage_ParseFileName(t *testing.T) {
+	for _, c := range cases {
+		for _, name := range append([]string{c.name}, c.oldName...) {
+			f := new(file)
+			if !f.parse(name) {
+				t.Errorf("cannot parse filename '%s'", name)
+				continue
+			}
+			if f.Type() != c.ftype {
+				t.Errorf("filename '%s' invalid type got '%d', want '%d'", name, f.Type(), c.ftype)
+			}
+			if f.Num() != c.num {
+				t.Errorf("filename '%s' invalid number got '%d', want '%d'", name, f.Num(), c.num)
+			}
+		}
+	}
+}
+
+func TestFileStorage_InvalidFileName(t *testing.T) {
+	for _, name := range invalidCases {
+		f := new(file)
+		if f.parse(name) {
+			t.Errorf("filename '%s' should be invalid", name)
+		}
+	}
+}
+
+func TestFileStorage_Locking(t *testing.T) {
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestfd-%d", os.Getuid()))
+
+	_, err := os.Stat(path)
+	if err == nil {
+		err = os.RemoveAll(path)
+		if err != nil {
+			t.Fatal("RemoveAll: got error: ", err)
+		}
+	}
+
+	p1, err := OpenFile(path)
+	if err != nil {
+		t.Fatal("OpenFile(1): got error: ", err)
+	}
+
+	defer os.RemoveAll(path)
+
+	p2, err := OpenFile(path)
+	if err != nil {
+		t.Logf("OpenFile(2): got error: %s (expected)", err)
+	} else {
+		p2.Close()
+		p1.Close()
+		t.Fatal("OpenFile(2): expect error")
+	}
+
+	p1.Close()
+
+	p3, err := OpenFile(path)
+	if err != nil {
+		t.Fatal("OpenFile(3): got error: ", err)
+	}
+	defer p3.Close()
+
+	l, err := p3.Lock()
+	if err != nil {
+		t.Fatal("storage lock failed(1): ", err)
+	}
+	_, err = p3.Lock()
+	if err == nil {
+		t.Fatal("expect error for second storage lock attempt")
+	} else {
+		t.Logf("storage lock got error: %s (expected)", err)
+	}
+	l.Release()
+	_, err = p3.Lock()
+	if err != nil {
+		t.Fatal("storage lock failed(2): ", err)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build darwin freebsd linux netbsd openbsd
+
+package storage
+
+import (
+	"os"
+	"syscall"
+)
+
+type unixFileLock struct {
+	f *os.File
+}
+
+func (fl *unixFileLock) release() error {
+	if err := setFileLock(fl.f, false); err != nil {
+		return err
+	}
+	return fl.f.Close()
+}
+
+func newFileLock(path string) (fl fileLock, err error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
+	if err != nil {
+		return
+	}
+	err = setFileLock(f, true)
+	if err != nil {
+		f.Close()
+		return
+	}
+	fl = &unixFileLock{f: f}
+	return
+}
+
+func setFileLock(f *os.File, lock bool) error {
+	how := syscall.LOCK_UN
+	if lock {
+		how = syscall.LOCK_EX
+	}
+	return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
+}
+
+func rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+var (
+	modkernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+	procMoveFileExW = modkernel32.NewProc("MoveFileExW")
+)
+
+const (
+	_MOVEFILE_REPLACE_EXISTING = 1
+)
+
+type windowsFileLock struct {
+	fd syscall.Handle
+}
+
+func (fl *windowsFileLock) release() error {
+	return syscall.Close(fl.fd)
+}
+
+func newFileLock(path string) (fl fileLock, err error) {
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+	if err != nil {
+		return
+	}
+	fl = &windowsFileLock{fd: fd}
+	return
+}
+
+func moveFileEx(from *uint16, to *uint16, flags uint32) error {
+	r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags))
+	if r1 == 0 {
+		if e1 != 0 {
+			return error(e1)
+		} else {
+			return syscall.EINVAL
+		}
+	}
+	return nil
+}
+
+func rename(oldpath, newpath string) error {
+	from, err := syscall.UTF16PtrFromString(oldpath)
+	if err != nil {
+		return err
+	}
+	to, err := syscall.UTF16PtrFromString(newpath)
+	if err != nil {
+		return err
+	}
+	return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go

@@ -0,0 +1,203 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"bytes"
+	"os"
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const typeShift = 3
+
+type memStorageLock struct {
+	ms *memStorage
+}
+
+func (lock *memStorageLock) Release() {
+	ms := lock.ms
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	if ms.slock == lock {
+		ms.slock = nil
+	}
+	return
+}
+
+// memStorage is a memory-backed storage.
+type memStorage struct {
+	mu       sync.Mutex
+	slock    *memStorageLock
+	files    map[uint64]*memFile
+	manifest *memFilePtr
+}
+
+// NewMemStorage returns a new memory-backed storage implementation.
+func NewMemStorage() Storage {
+	return &memStorage{
+		files: make(map[uint64]*memFile),
+	}
+}
+
+func (ms *memStorage) Lock() (util.Releaser, error) {
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	if ms.slock != nil {
+		return nil, ErrLocked
+	}
+	ms.slock = &memStorageLock{ms: ms}
+	return ms.slock, nil
+}
+
+func (*memStorage) Log(str string) {}
+
+func (ms *memStorage) GetFile(num uint64, t FileType) File {
+	return &memFilePtr{ms: ms, num: num, t: t}
+}
+
+func (ms *memStorage) GetFiles(t FileType) ([]File, error) {
+	ms.mu.Lock()
+	var ff []File
+	for x, _ := range ms.files {
+		num, mt := x>>typeShift, FileType(x)&TypeAll
+		if mt&t == 0 {
+			continue
+		}
+		ff = append(ff, &memFilePtr{ms: ms, num: num, t: mt})
+	}
+	ms.mu.Unlock()
+	return ff, nil
+}
+
+func (ms *memStorage) GetManifest() (File, error) {
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	if ms.manifest == nil {
+		return nil, os.ErrNotExist
+	}
+	return ms.manifest, nil
+}
+
+func (ms *memStorage) SetManifest(f File) error {
+	fm, ok := f.(*memFilePtr)
+	if !ok || fm.t != TypeManifest {
+		return ErrInvalidFile
+	}
+	ms.mu.Lock()
+	ms.manifest = fm
+	ms.mu.Unlock()
+	return nil
+}
+
+func (*memStorage) Close() error { return nil }
+
+type memReader struct {
+	*bytes.Reader
+	m *memFile
+}
+
+func (mr *memReader) Close() error {
+	return mr.m.Close()
+}
+
+type memFile struct {
+	bytes.Buffer
+	ms   *memStorage
+	open bool
+}
+
+func (*memFile) Sync() error { return nil }
+func (m *memFile) Close() error {
+	m.ms.mu.Lock()
+	m.open = false
+	m.ms.mu.Unlock()
+	return nil
+}
+
+type memFilePtr struct {
+	ms  *memStorage
+	num uint64
+	t   FileType
+}
+
+func (p *memFilePtr) x() uint64 {
+	return p.Num()<<typeShift | uint64(p.Type())
+}
+
+func (p *memFilePtr) Open() (Reader, error) {
+	ms := p.ms
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	if m, exist := ms.files[p.x()]; exist {
+		if m.open {
+			return nil, errFileOpen
+		}
+		m.open = true
+		return &memReader{Reader: bytes.NewReader(m.Bytes()), m: m}, nil
+	}
+	return nil, os.ErrNotExist
+}
+
+func (p *memFilePtr) Create() (Writer, error) {
+	ms := p.ms
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	m, exist := ms.files[p.x()]
+	if exist {
+		if m.open {
+			return nil, errFileOpen
+		}
+		m.Reset()
+	} else {
+		m = &memFile{ms: ms}
+		ms.files[p.x()] = m
+	}
+	m.open = true
+	return m, nil
+}
+
+func (p *memFilePtr) Replace(newfile File) error {
+	p1, ok := newfile.(*memFilePtr)
+	if !ok {
+		return ErrInvalidFile
+	}
+	ms := p.ms
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	m1, exist := ms.files[p1.x()]
+	if !exist {
+		return os.ErrNotExist
+	}
+	m0, exist := ms.files[p.x()]
+	if (exist && m0.open) || m1.open {
+		return errFileOpen
+	}
+	delete(ms.files, p1.x())
+	ms.files[p.x()] = m1
+	return nil
+}
+
+func (p *memFilePtr) Type() FileType {
+	return p.t
+}
+
+func (p *memFilePtr) Num() uint64 {
+	return p.num
+}
+
+func (p *memFilePtr) Remove() error {
+	ms := p.ms
+	ms.mu.Lock()
+	defer ms.mu.Unlock()
+	if _, exist := ms.files[p.x()]; exist {
+		delete(ms.files, p.x())
+		return nil
+	}
+	return os.ErrNotExist
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package storage
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestMemStorage(t *testing.T) {
+	m := NewMemStorage()
+
+	l, err := m.Lock()
+	if err != nil {
+		t.Fatal("storage lock failed(1): ", err)
+	}
+	_, err = m.Lock()
+	if err == nil {
+		t.Fatal("expect error for second storage lock attempt")
+	} else {
+		t.Logf("storage lock got error: %s (expected)", err)
+	}
+	l.Release()
+	_, err = m.Lock()
+	if err != nil {
+		t.Fatal("storage lock failed(2): ", err)
+	}
+
+	f := m.GetFile(1, TypeTable)
+	if f.Num() != 1 && f.Type() != TypeTable {
+		t.Fatal("invalid file number and type")
+	}
+	w, _ := f.Create()
+	w.Write([]byte("abc"))
+	w.Close()
+	if ff, _ := m.GetFiles(TypeAll); len(ff) != 1 {
+		t.Fatal("invalid GetFiles len")
+	}
+	buf := new(bytes.Buffer)
+	r, err := f.Open()
+	if err != nil {
+		t.Fatal("Open: got error: ", err)
+	}
+	buf.ReadFrom(r)
+	r.Close()
+	if got := buf.String(); got != "abc" {
+		t.Fatalf("Read: invalid value, want=abc got=%s", got)
+	}
+	if _, err := f.Open(); err != nil {
+		t.Fatal("Open: got error: ", err)
+	}
+	if _, err := m.GetFile(1, TypeTable).Open(); err == nil {
+		t.Fatal("expecting error")
+	}
+	f.Remove()
+	if ff, _ := m.GetFiles(TypeAll); len(ff) != 0 {
+		t.Fatal("invalid GetFiles len", len(ff))
+	}
+	if _, err := f.Open(); err == nil {
+		t.Fatal("expecting error")
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package storage provides storage abstraction for LevelDB.
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type FileType uint32
+
+const (
+	TypeManifest FileType = 1 << iota
+	TypeJournal
+	TypeTable
+	TypeTemp
+
+	TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp
+)
+
+func (t FileType) String() string {
+	switch t {
+	case TypeManifest:
+		return "manifest"
+	case TypeJournal:
+		return "journal"
+	case TypeTable:
+		return "table"
+	case TypeTemp:
+		return "temp"
+	}
+	return fmt.Sprintf("<unknown:%d>", t)
+}
+
+var (
+	ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument")
+	ErrLocked      = errors.New("leveldb/storage: already locked")
+	ErrClosed      = errors.New("leveldb/storage: closed")
+)
+
+// Syncer is the interface that wraps basic Sync method.
+type Syncer interface {
+	// Sync commits the current contents of the file to stable storage.
+	Sync() error
+}
+
+// Reader is the interface that groups the basic Read, Seek, ReadAt and Close
+// methods.
+type Reader interface {
+	io.ReadSeeker
+	io.ReaderAt
+	io.Closer
+}
+
+// Writer is the interface that groups the basic Write, Sync and Close
+// methods.
+type Writer interface {
+	io.WriteCloser
+	Syncer
+}
+
+// File is the file.
+type File interface {
+	// Open opens the file for read. Returns os.ErrNotExist error
+	// if the file does not exist.
+	// Returns ErrClosed if the underlying storage is closed.
+	Open() (r Reader, err error)
+
+	// Create creates the file for writting. Truncate the file if
+	// already exist.
+	// Returns ErrClosed if the underlying storage is closed.
+	Create() (w Writer, err error)
+
+	// Replace replaces file with newfile.
+	// Returns ErrClosed if the underlying storage is closed.
+	Replace(newfile File) error
+
+	// Type returns the file type
+	Type() FileType
+
+	// Num returns the file number.
+	Num() uint64
+
+	// Remove removes the file.
+	// Returns ErrClosed if the underlying storage is closed.
+	Remove() error
+}
+
+// Storage is the storage.
+type Storage interface {
+	// Lock locks the storage. Any subsequent attempt to call Lock will fail
+	// until the last lock released.
+	// After use the caller should call the Release method.
+	Lock() (l util.Releaser, err error)
+
+	// Log logs a string. This is used for logging. An implementation
+	// may write to a file, stdout or simply do nothing.
+	Log(str string)
+
+	// GetFile returns a file for the given number and type. GetFile will never
+	// returns nil, even if the underlying storage is closed.
+	GetFile(num uint64, t FileType) File
+
+	// GetFiles returns a slice of files that match the given file types.
+	// The file types may be OR'ed together.
+	GetFiles(t FileType) ([]File, error)
+
+	// GetManifest returns a manifest file. Returns os.ErrNotExist if manifest
+	// file does not exist.
+	GetManifest() (File, error)
+
+	// SetManifest sets the given file as manifest file. The given file should
+	// be a manifest file type or error will be returned.
+	SetManifest(f File) error
+
+	// Close closes the storage. It is valid to call Close multiple times.
+	// Other methods should not be called after the storage has been closed.
+	Close() error
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go

@@ -0,0 +1,459 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENE file.
+
+package leveldb
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const typeShift = 4
+
+var (
+	tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
+	tsErrFileOpen    = errors.New("leveldb.testStorage: file still open")
+)
+
+var (
+	tsFSEnv  = os.Getenv("GOLEVELDB_USEFS")
+	tsKeepFS = tsFSEnv == "2"
+	tsFS     = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
+	tsMU     = &sync.Mutex{}
+	tsNum    = 0
+)
+
+type tsLock struct {
+	ts *testStorage
+	r  util.Releaser
+}
+
+func (l tsLock) Release() {
+	l.r.Release()
+	l.ts.t.Log("I: storage lock released")
+}
+
+type tsReader struct {
+	tf tsFile
+	storage.Reader
+}
+
+func (tr tsReader) Read(b []byte) (n int, err error) {
+	ts := tr.tf.ts
+	ts.countRead(tr.tf.Type())
+	n, err = tr.Reader.Read(b)
+	if err != nil && err != io.EOF {
+		ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
+	}
+	return
+}
+
+func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
+	ts := tr.tf.ts
+	ts.countRead(tr.tf.Type())
+	n, err = tr.Reader.ReadAt(b, off)
+	if err != nil && err != io.EOF {
+		ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
+	}
+	return
+}
+
+func (tr tsReader) Close() (err error) {
+	err = tr.Reader.Close()
+	tr.tf.close("reader", err)
+	return
+}
+
+type tsWriter struct {
+	tf tsFile
+	storage.Writer
+}
+
+func (tw tsWriter) Write(b []byte) (n int, err error) {
+	ts := tw.tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	if ts.emuWriteErr&tw.tf.Type() != 0 {
+		return 0, errors.New("leveldb.testStorage: emulated write error")
+	}
+	n, err = tw.Writer.Write(b)
+	if err != nil {
+		ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
+	}
+	return
+}
+
+func (tw tsWriter) Sync() (err error) {
+	ts := tw.tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	for ts.emuDelaySync&tw.tf.Type() != 0 {
+		ts.cond.Wait()
+	}
+	if ts.emuSyncErr&tw.tf.Type() != 0 {
+		return errors.New("leveldb.testStorage: emulated sync error")
+	}
+	err = tw.Writer.Sync()
+	if err != nil {
+		ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
+	}
+	return
+}
+
+func (tw tsWriter) Close() (err error) {
+	err = tw.Writer.Close()
+	tw.tf.close("reader", err)
+	return
+}
+
+type tsFile struct {
+	ts *testStorage
+	storage.File
+}
+
+func (tf tsFile) x() uint64 {
+	return tf.Num()<<typeShift | uint64(tf.Type())
+}
+
+func (tf tsFile) checkOpen(m string) error {
+	ts := tf.ts
+	if writer, ok := ts.opens[tf.x()]; ok {
+		if writer {
+			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
+		} else {
+			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
+		}
+		return tsErrFileOpen
+	}
+	return nil
+}
+
+func (tf tsFile) close(m string, err error) {
+	ts := tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	if _, ok := ts.opens[tf.x()]; !ok {
+		ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
+	} else if err == nil {
+		ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
+	}
+	delete(ts.opens, tf.x())
+	if err != nil {
+		ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
+	}
+}
+
+func (tf tsFile) Open() (r storage.Reader, err error) {
+	ts := tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	err = tf.checkOpen("open")
+	if err != nil {
+		return
+	}
+	if ts.emuOpenErr&tf.Type() != 0 {
+		err = errors.New("leveldb.testStorage: emulated open error")
+		return
+	}
+	r, err = tf.File.Open()
+	if err != nil {
+		if ts.ignoreOpenErr&tf.Type() != 0 {
+			ts.t.Logf("I: cannot open file, num=%d type=%v: %v (ignored)", tf.Num(), tf.Type(), err)
+		} else {
+			ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
+		}
+	} else {
+		ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
+		ts.opens[tf.x()] = false
+		r = tsReader{tf, r}
+	}
+	return
+}
+
+func (tf tsFile) Create() (w storage.Writer, err error) {
+	ts := tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	err = tf.checkOpen("create")
+	if err != nil {
+		return
+	}
+	if ts.emuCreateErr&tf.Type() != 0 {
+		err = errors.New("leveldb.testStorage: emulated create error")
+		return
+	}
+	w, err = tf.File.Create()
+	if err != nil {
+		ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
+	} else {
+		ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
+		ts.opens[tf.x()] = true
+		w = tsWriter{tf, w}
+	}
+	return
+}
+
+func (tf tsFile) Remove() (err error) {
+	ts := tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	err = tf.checkOpen("remove")
+	if err != nil {
+		return
+	}
+	err = tf.File.Remove()
+	if err != nil {
+		ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
+	} else {
+		ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
+	}
+	return
+}
+
+type testStorage struct {
+	t *testing.T
+	storage.Storage
+	closeFn func() error
+
+	mu   sync.Mutex
+	cond sync.Cond
+	// Open files, true=writer, false=reader
+	opens         map[uint64]bool
+	emuOpenErr    storage.FileType
+	emuCreateErr  storage.FileType
+	emuDelaySync  storage.FileType
+	emuWriteErr   storage.FileType
+	emuSyncErr    storage.FileType
+	ignoreOpenErr storage.FileType
+	readCnt       uint64
+	readCntEn     storage.FileType
+}
+
+func (ts *testStorage) SetOpenErr(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuOpenErr = t
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetCreateErr(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuCreateErr = t
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) DelaySync(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuDelaySync |= t
+	ts.cond.Broadcast()
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) ReleaseSync(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuDelaySync &= ^t
+	ts.cond.Broadcast()
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetWriteErr(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuWriteErr = t
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetSyncErr(t storage.FileType) {
+	ts.mu.Lock()
+	ts.emuSyncErr = t
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) ReadCounter() uint64 {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	return ts.readCnt
+}
+
+func (ts *testStorage) ResetReadCounter() {
+	ts.mu.Lock()
+	ts.readCnt = 0
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetReadCounter(t storage.FileType) {
+	ts.mu.Lock()
+	ts.readCntEn = t
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) countRead(t storage.FileType) {
+	ts.mu.Lock()
+	if ts.readCntEn&t != 0 {
+		ts.readCnt++
+	}
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetIgnoreOpenErr(t storage.FileType) {
+	ts.ignoreOpenErr = t
+}
+
+func (ts *testStorage) Lock() (r util.Releaser, err error) {
+	r, err = ts.Storage.Lock()
+	if err != nil {
+		ts.t.Logf("W: storage locking failed: %v", err)
+	} else {
+		ts.t.Log("I: storage locked")
+		r = tsLock{ts, r}
+	}
+	return
+}
+
+func (ts *testStorage) Log(str string) {
+	ts.t.Log("L: " + str)
+	ts.Storage.Log(str)
+}
+
+func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
+	return tsFile{ts, ts.Storage.GetFile(num, t)}
+}
+
+func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
+	ff0, err := ts.Storage.GetFiles(t)
+	if err != nil {
+		ts.t.Errorf("E: get files failed: %v", err)
+		return
+	}
+	ff = make([]storage.File, len(ff0))
+	for i, f := range ff0 {
+		ff[i] = tsFile{ts, f}
+	}
+	ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
+	return
+}
+
+func (ts *testStorage) GetManifest() (f storage.File, err error) {
+	f0, err := ts.Storage.GetManifest()
+	if err != nil {
+		if !os.IsNotExist(err) {
+			ts.t.Errorf("E: get manifest failed: %v", err)
+		}
+		return
+	}
+	f = tsFile{ts, f0}
+	ts.t.Logf("I: get manifest, num=%d", f.Num())
+	return
+}
+
+func (ts *testStorage) SetManifest(f storage.File) error {
+	tf, ok := f.(tsFile)
+	if !ok {
+		ts.t.Error("E: set manifest failed: type assertion failed")
+		return tsErrInvalidFile
+	} else if tf.Type() != storage.TypeManifest {
+		ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
+		return tsErrInvalidFile
+	}
+	err := ts.Storage.SetManifest(tf.File)
+	if err != nil {
+		ts.t.Errorf("E: set manifest failed: %v", err)
+	} else {
+		ts.t.Logf("I: set manifest, num=%d", tf.Num())
+	}
+	return err
+}
+
+func (ts *testStorage) Close() error {
+	ts.CloseCheck()
+	err := ts.Storage.Close()
+	if err != nil {
+		ts.t.Errorf("E: closing storage failed: %v", err)
+	} else {
+		ts.t.Log("I: storage closed")
+	}
+	if ts.closeFn != nil {
+		if err := ts.closeFn(); err != nil {
+			ts.t.Errorf("E: close function: %v", err)
+		}
+	}
+	return err
+}
+
+func (ts *testStorage) CloseCheck() {
+	ts.mu.Lock()
+	if len(ts.opens) == 0 {
+		ts.t.Log("I: all files are closed")
+	} else {
+		ts.t.Errorf("E: %d files still open", len(ts.opens))
+		for x, writer := range ts.opens {
+			num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
+			ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
+		}
+	}
+	ts.mu.Unlock()
+}
+
+func newTestStorage(t *testing.T) *testStorage {
+	var stor storage.Storage
+	var closeFn func() error
+	if tsFS {
+		for {
+			tsMU.Lock()
+			num := tsNum
+			tsNum++
+			tsMU.Unlock()
+			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
+			if _, err := os.Stat(path); err != nil {
+				stor, err = storage.OpenFile(path)
+				if err != nil {
+					t.Fatalf("F: cannot create storage: %v", err)
+				}
+				t.Logf("I: storage created: %s", path)
+				closeFn = func() error {
+					for _, name := range []string{"LOG.old", "LOG"} {
+						f, err := os.Open(filepath.Join(path, name))
+						if err != nil {
+							continue
+						}
+						if log, err := ioutil.ReadAll(f); err != nil {
+							t.Logf("---------------------- %s ----------------------", name)
+							t.Logf("cannot read log: %v", err)
+							t.Logf("---------------------- %s ----------------------", name)
+						} else if len(log) > 0 {
+							t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
+							t.Logf("---------------------- %s ----------------------", name)
+						}
+						f.Close()
+					}
+					if tsKeepFS {
+						return nil
+					}
+					return os.RemoveAll(path)
+				}
+
+				break
+			}
+		}
+	} else {
+		stor = storage.NewMemStorage()
+	}
+	ts := &testStorage{
+		t:       t,
+		Storage: stor,
+		closeFn: closeFn,
+		opens:   make(map[uint64]bool),
+	}
+	ts.cond.L = &ts.mu
+	return ts
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -0,0 +1,426 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"sort"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/table"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// table file
+type tFile struct {
+	file     storage.File
+	seekLeft int32
+	size     uint64
+	min, max iKey
+}
+
+// test if key is after t
+func (t *tFile) isAfter(key []byte, ucmp comparer.BasicComparer) bool {
+	return key != nil && ucmp.Compare(key, t.max.ukey()) > 0
+}
+
+// test if key is before t
+func (t *tFile) isBefore(key []byte, ucmp comparer.BasicComparer) bool {
+	return key != nil && ucmp.Compare(key, t.min.ukey()) < 0
+}
+
+func (t *tFile) incrSeek() int32 {
+	return atomic.AddInt32(&t.seekLeft, -1)
+}
+
+func newTFile(file storage.File, size uint64, min, max iKey) *tFile {
+	f := &tFile{
+		file: file,
+		size: size,
+		min:  min,
+		max:  max,
+	}
+
+	// We arrange to automatically compact this file after
+	// a certain number of seeks.  Let's assume:
+	//   (1) One seek costs 10ms
+	//   (2) Writing or reading 1MB costs 10ms (100MB/s)
+	//   (3) A compaction of 1MB does 25MB of IO:
+	//         1MB read from this level
+	//         10-12MB read from next level (boundaries may be misaligned)
+	//         10-12MB written to next level
+	// This implies that 25 seeks cost the same as the compaction
+	// of 1MB of data.  I.e., one seek costs approximately the
+	// same as the compaction of 40KB of data.  We are a little
+	// conservative and allow approximately one seek for every 16KB
+	// of data before triggering a compaction.
+	f.seekLeft = int32(size / 16384)
+	if f.seekLeft < 100 {
+		f.seekLeft = 100
+	}
+
+	return f
+}
+
+// table files
+type tFiles []*tFile
+
+func (tf tFiles) Len() int      { return len(tf) }
+func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
+
+func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
+	a, b := tf[i], tf[j]
+	n := icmp.Compare(a.min, b.min)
+	if n == 0 {
+		return a.file.Num() < b.file.Num()
+	}
+	return n < 0
+}
+
+func (tf tFiles) lessByNum(i, j int) bool {
+	return tf[i].file.Num() > tf[j].file.Num()
+}
+
+func (tf tFiles) sortByKey(icmp *iComparer) {
+	sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp})
+}
+
+func (tf tFiles) sortByNum() {
+	sort.Sort(&tFilesSortByNum{tFiles: tf})
+}
+
+func (tf tFiles) size() (sum uint64) {
+	for _, t := range tf {
+		sum += t.size
+	}
+	return sum
+}
+
+func (tf tFiles) searchMin(key iKey, icmp *iComparer) int {
+	return sort.Search(len(tf), func(i int) bool {
+		return icmp.Compare(tf[i].min, key) >= 0
+	})
+}
+
+func (tf tFiles) searchMax(key iKey, icmp *iComparer) int {
+	return sort.Search(len(tf), func(i int) bool {
+		return icmp.Compare(tf[i].max, key) >= 0
+	})
+}
+
+func (tf tFiles) isOverlaps(min, max []byte, disjSorted bool, icmp *iComparer) bool {
+	ucmp := icmp.cmp
+
+	if !disjSorted {
+		// Need to check against all files
+		for _, t := range tf {
+			if !t.isAfter(min, ucmp) && !t.isBefore(max, ucmp) {
+				return true
+			}
+		}
+		return false
+	}
+
+	var idx int
+	if len(min) > 0 {
+		// Find the earliest possible internal key for min
+		idx = tf.searchMax(newIKey(min, kMaxSeq, tSeek), icmp)
+	}
+
+	if idx >= len(tf) {
+		// beginning of range is after all files, so no overlap
+		return false
+	}
+	return !tf[idx].isBefore(max, ucmp)
+}
+
+func (tf tFiles) getOverlaps(min, max []byte, r *tFiles, disjSorted bool, ucmp comparer.BasicComparer) {
+	for i := 0; i < len(tf); {
+		t := tf[i]
+		i++
+		if t.isAfter(min, ucmp) || t.isBefore(max, ucmp) {
+			continue
+		}
+
+		*r = append(*r, t)
+		if !disjSorted {
+			// Level-0 files may overlap each other.  So check if the newly
+			// added file has expanded the range.  If so, restart search.
+			if min != nil && ucmp.Compare(t.min.ukey(), min) < 0 {
+				min = t.min.ukey()
+				*r = nil
+				i = 0
+			} else if max != nil && ucmp.Compare(t.max.ukey(), max) > 0 {
+				max = t.max.ukey()
+				*r = nil
+				i = 0
+			}
+		}
+	}
+
+	return
+}
+
+func (tf tFiles) getRange(icmp *iComparer) (min, max iKey) {
+	for i, t := range tf {
+		if i == 0 {
+			min, max = t.min, t.max
+			continue
+		}
+		if icmp.Compare(t.min, min) < 0 {
+			min = t.min
+		}
+		if icmp.Compare(t.max, max) > 0 {
+			max = t.max
+		}
+	}
+
+	return
+}
+
+func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer {
+	if slice != nil {
+		var start, limit int
+		if slice.Start != nil {
+			start = tf.searchMax(iKey(slice.Start), icmp)
+		}
+		if slice.Limit != nil {
+			limit = tf.searchMin(iKey(slice.Limit), icmp)
+		} else {
+			limit = tf.Len()
+		}
+		tf = tf[start:limit]
+	}
+	return iterator.NewArrayIndexer(&tFilesArrayIndexer{
+		tFiles: tf,
+		tops:   tops,
+		icmp:   icmp,
+		slice:  slice,
+		ro:     ro,
+	})
+}
+
+type tFilesArrayIndexer struct {
+	tFiles
+	tops  *tOps
+	icmp  *iComparer
+	slice *util.Range
+	ro    *opt.ReadOptions
+}
+
+func (a *tFilesArrayIndexer) Search(key []byte) int {
+	return a.searchMax(iKey(key), a.icmp)
+}
+
+func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
+	if i == 0 || i == a.Len()-1 {
+		return a.tops.newIterator(a.tFiles[i], a.slice, a.ro)
+	}
+	return a.tops.newIterator(a.tFiles[i], nil, a.ro)
+}
+
+type tFilesSortByKey struct {
+	tFiles
+	icmp *iComparer
+}
+
+func (x *tFilesSortByKey) Less(i, j int) bool {
+	return x.lessByKey(x.icmp, i, j)
+}
+
+type tFilesSortByNum struct {
+	tFiles
+}
+
+func (x *tFilesSortByNum) Less(i, j int) bool {
+	return x.lessByNum(i, j)
+}
+
+// table operations
+type tOps struct {
+	s       *session
+	cache   cache.Cache
+	cacheNS cache.Namespace
+}
+
+func newTableOps(s *session, cacheCap int) *tOps {
+	c := cache.NewLRUCache(cacheCap)
+	ns := c.GetNamespace(0)
+	return &tOps{s, c, ns}
+}
+
+func (t *tOps) create() (*tWriter, error) {
+	file := t.s.getTableFile(t.s.allocFileNum())
+	fw, err := file.Create()
+	if err != nil {
+		return nil, err
+	}
+	return &tWriter{
+		t:    t,
+		file: file,
+		w:    fw,
+		tw:   table.NewWriter(fw, t.s.o),
+	}, nil
+}
+
+func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
+	w, err := t.create()
+	if err != nil {
+		return f, n, err
+	}
+
+	defer func() {
+		if err != nil {
+			w.drop()
+		}
+	}()
+
+	for src.Next() {
+		err = w.add(src.Key(), src.Value())
+		if err != nil {
+			return
+		}
+	}
+	err = src.Error()
+	if err != nil {
+		return
+	}
+
+	n = w.tw.EntriesLen()
+	f, err = w.finish()
+	return
+}
+
+func (t *tOps) lookup(f *tFile) (c cache.Object, err error) {
+	num := f.file.Num()
+	c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+		var r storage.Reader
+		r, err = f.file.Open()
+		if err != nil {
+			return
+		}
+
+		o := t.s.o
+
+		var cacheNS cache.Namespace
+		if bc := o.GetBlockCache(); bc != nil {
+			cacheNS = bc.GetNamespace(num)
+		}
+
+		ok = true
+		value = table.NewReader(r, int64(f.size), cacheNS, o)
+		charge = 1
+		fin = func() {
+			r.Close()
+		}
+		return
+	})
+	if !ok && err == nil {
+		err = ErrClosed
+	}
+	return
+}
+
+func (t *tOps) get(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
+	c, err := t.lookup(f)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer c.Release()
+	return c.Value().(*table.Reader).Find(key, ro)
+}
+
+func (t *tOps) getApproximateOffset(f *tFile, key []byte) (offset uint64, err error) {
+	c, err := t.lookup(f)
+	if err != nil {
+		return
+	}
+	_offset, err := c.Value().(*table.Reader).GetApproximateOffset(key)
+	offset = uint64(_offset)
+	c.Release()
+	return
+}
+
+func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	c, err := t.lookup(f)
+	if err != nil {
+		return iterator.NewEmptyIterator(err)
+	}
+	iter := c.Value().(*table.Reader).NewIterator(slice, ro)
+	iter.SetReleaser(c)
+	return iter
+}
+
+func (t *tOps) remove(f *tFile) {
+	num := f.file.Num()
+	t.cacheNS.Delete(num, func(exist bool) {
+		if err := f.file.Remove(); err != nil {
+			t.s.logf("table@remove removing @%d %q", num, err)
+		} else {
+			t.s.logf("table@remove removed @%d", num)
+		}
+		if bc := t.s.o.GetBlockCache(); bc != nil {
+			bc.GetNamespace(num).Zap(false)
+		}
+	})
+}
+
+func (t *tOps) close() {
+	t.cache.Zap(true)
+}
+
+type tWriter struct {
+	t *tOps
+
+	file storage.File
+	w    storage.Writer
+	tw   *table.Writer
+
+	first, last []byte
+}
+
+func (w *tWriter) add(key, value []byte) error {
+	if w.first == nil {
+		w.first = append([]byte{}, key...)
+	}
+	w.last = append(w.last[:0], key...)
+	return w.tw.Append(key, value)
+}
+
+func (w *tWriter) empty() bool {
+	return w.first == nil
+}
+
+func (w *tWriter) finish() (f *tFile, err error) {
+	err = w.tw.Close()
+	if err != nil {
+		return
+	}
+	err = w.w.Sync()
+	if err != nil {
+		w.w.Close()
+		return
+	}
+	w.w.Close()
+	f = newTFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
+	return
+}
+
+func (w *tWriter) drop() {
+	w.w.Close()
+	w.file.Remove()
+	w.t.s.reuseFileNum(w.file.Num())
+	w.w = nil
+	w.file = nil
+	w.tw = nil
+	w.first = nil
+	w.last = nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package table
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func (b *block) TestNewIterator(slice *util.Range) iterator.Iterator {
+	return b.newIterator(slice, false, nil)
+}
+
+var _ = testutil.Defer(func() {
+	Describe("Block", func() {
+		Build := func(kv *testutil.KeyValue, restartInterval int) *block {
+			// Building the block.
+			bw := &blockWriter{
+				restartInterval: restartInterval,
+				scratch:         make([]byte, 30),
+			}
+			kv.Iterate(func(i int, key, value []byte) {
+				bw.append(key, value)
+			})
+			bw.finish()
+
+			// Opening the block.
+			data := bw.buf.Bytes()
+			restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
+			return &block{
+				cmp:            comparer.DefaultComparer,
+				data:           data,
+				restartsLen:    restartsLen,
+				restartsOffset: len(data) - (restartsLen+1)*4,
+			}
+		}
+
+		Describe("read test", func() {
+			for restartInterval := 1; restartInterval <= 5; restartInterval++ {
+				Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
+					kv := &testutil.KeyValue{}
+					Text := func() string {
+						return fmt.Sprintf("and %d keys", kv.Len())
+					}
+
+					Test := func() {
+						// Make block.
+						br := Build(kv, restartInterval)
+						// Do testing.
+						testutil.KeyValueTesting(nil, br, kv.Clone())
+					}
+
+					Describe(Text(), Test)
+
+					kv.PutString("", "empty")
+					Describe(Text(), Test)
+
+					kv.PutString("a1", "foo")
+					Describe(Text(), Test)
+
+					kv.PutString("a2", "v")
+					Describe(Text(), Test)
+
+					kv.PutString("a3qqwrkks", "hello")
+					Describe(Text(), Test)
+
+					kv.PutString("a4", "bar")
+					Describe(Text(), Test)
+
+					kv.PutString("a5111111", "v5")
+					kv.PutString("a6", "")
+					kv.PutString("a7", "v7")
+					kv.PutString("a8", "vvvvvvvvvvvvvvvvvvvvvv8")
+					kv.PutString("b", "v9")
+					kv.PutString("c9", "v9")
+					kv.PutString("c91", "v9")
+					kv.PutString("d0", "v9")
+					Describe(Text(), Test)
+				})
+			}
+		})
+
+		Describe("out-of-bound slice test", func() {
+			kv := &testutil.KeyValue{}
+			kv.PutString("k1", "v1")
+			kv.PutString("k2", "v2")
+			kv.PutString("k3abcdefgg", "v3")
+			kv.PutString("k4", "v4")
+			kv.PutString("k5", "v5")
+			for restartInterval := 1; restartInterval <= 5; restartInterval++ {
+				Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
+					// Make block.
+					br := Build(kv, restartInterval)
+
+					Test := func(r *util.Range) func(done Done) {
+						return func(done Done) {
+							iter := br.newIterator(r, false, nil)
+							Expect(iter.Error()).ShouldNot(HaveOccurred())
+
+							t := testutil.IteratorTesting{
+								KeyValue: kv.Clone(),
+								Iter:     iter,
+							}
+
+							testutil.DoIteratorTesting(&t)
+							done <- true
+						}
+					}
+
+					It("Should do iterations and seeks correctly #0",
+						Test(&util.Range{Start: []byte("k0"), Limit: []byte("k6")}), 2.0)
+
+					It("Should do iterations and seeks correctly #1",
+						Test(&util.Range{Start: []byte(""), Limit: []byte("zzzzzzz")}), 2.0)
+				})
+			}
+		})
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go

@@ -0,0 +1,848 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package table
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"code.google.com/p/snappy-go/snappy"
+
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	ErrNotFound     = util.ErrNotFound
+	ErrIterReleased = errors.New("leveldb/table: iterator released")
+)
+
+func max(x, y int) int {
+	if x > y {
+		return x
+	}
+	return y
+}
+
+type block struct {
+	cmp            comparer.BasicComparer
+	data           []byte
+	restartsLen    int
+	restartsOffset int
+	// Whether checksum is verified and valid.
+	checksum bool
+}
+
+func (b *block) seek(rstart, rlimit int, key []byte) (index, offset int, err error) {
+	n := b.restartsOffset
+	data := b.data
+	cmp := b.cmp
+
+	index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
+		offset := int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):]))
+		offset += 1                               // shared always zero, since this is a restart point
+		v1, n1 := binary.Uvarint(data[offset:])   // key length
+		_, n2 := binary.Uvarint(data[offset+n1:]) // value length
+		m := offset + n1 + n2
+		return cmp.Compare(data[m:m+int(v1)], key) > 0
+	}) + rstart - 1
+	if index < rstart {
+		// The smallest key is greater-than key sought.
+		index = rstart
+	}
+	offset = int(binary.LittleEndian.Uint32(data[n+4*index:]))
+	return
+}
+
+func (b *block) restartIndex(rstart, rlimit, offset int) int {
+	n := b.restartsOffset
+	data := b.data
+	return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
+		return int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):])) > offset
+	}) + rstart - 1
+}
+
+func (b *block) restartOffset(index int) int {
+	return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:]))
+}
+
+func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) {
+	if offset >= b.restartsOffset {
+		if offset != b.restartsOffset {
+			err = errors.New("leveldb/table: Reader: BlockEntry: invalid block (block entries offset not aligned)")
+		}
+		return
+	}
+	v0, n0 := binary.Uvarint(b.data[offset:])       // Shared prefix length
+	v1, n1 := binary.Uvarint(b.data[offset+n0:])    // Key length
+	v2, n2 := binary.Uvarint(b.data[offset+n0+n1:]) // Value length
+	m := n0 + n1 + n2
+	n = m + int(v1) + int(v2)
+	if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset {
+		err = errors.New("leveldb/table: Reader: invalid block (block entries corrupted)")
+		return
+	}
+	key = b.data[offset+m : offset+m+int(v1)]
+	value = b.data[offset+m+int(v1) : offset+n]
+	nShared = int(v0)
+	return
+}
+
+func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releaser) *blockIter {
+	bi := &blockIter{
+		block: b,
+		cache: cache,
+		// Valid key should never be nil.
+		key:             make([]byte, 0),
+		dir:             dirSOI,
+		riStart:         0,
+		riLimit:         b.restartsLen,
+		offsetStart:     0,
+		offsetRealStart: 0,
+		offsetLimit:     b.restartsOffset,
+	}
+	if slice != nil {
+		if slice.Start != nil {
+			if bi.Seek(slice.Start) {
+				bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset)
+				bi.offsetStart = b.restartOffset(bi.riStart)
+				bi.offsetRealStart = bi.prevOffset
+			} else {
+				bi.riStart = b.restartsLen
+				bi.offsetStart = b.restartsOffset
+				bi.offsetRealStart = b.restartsOffset
+			}
+		}
+		if slice.Limit != nil {
+			if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) {
+				bi.offsetLimit = bi.prevOffset
+				bi.riLimit = bi.restartIndex + 1
+			}
+		}
+		bi.reset()
+		if bi.offsetStart > bi.offsetLimit {
+			bi.sErr(errors.New("leveldb/table: Reader: invalid slice range"))
+		}
+	}
+	return bi
+}
+
+type dir int
+
+const (
+	dirReleased dir = iota - 1
+	dirSOI
+	dirEOI
+	dirBackward
+	dirForward
+)
+
+type blockIter struct {
+	block           *block
+	cache, releaser util.Releaser
+	key, value      []byte
+	offset          int
+	// Previous offset, only filled by Next.
+	prevOffset   int
+	prevNode     []int
+	prevKeys     []byte
+	restartIndex int
+	// Iterator direction.
+	dir dir
+	// Restart index slice range.
+	riStart int
+	riLimit int
+	// Offset slice range.
+	offsetStart     int
+	offsetRealStart int
+	offsetLimit     int
+	// Error.
+	err error
+}
+
+func (i *blockIter) sErr(err error) {
+	i.err = err
+	i.key = nil
+	i.value = nil
+	i.prevNode = nil
+	i.prevKeys = nil
+}
+
+func (i *blockIter) reset() {
+	if i.dir == dirBackward {
+		i.prevNode = i.prevNode[:0]
+		i.prevKeys = i.prevKeys[:0]
+	}
+	i.restartIndex = i.riStart
+	i.offset = i.offsetStart
+	i.dir = dirSOI
+	i.key = i.key[:0]
+	i.value = nil
+}
+
+func (i *blockIter) isFirst() bool {
+	switch i.dir {
+	case dirForward:
+		return i.prevOffset == i.offsetRealStart
+	case dirBackward:
+		return len(i.prevNode) == 1 && i.restartIndex == i.riStart
+	}
+	return false
+}
+
+func (i *blockIter) isLast() bool {
+	switch i.dir {
+	case dirForward, dirBackward:
+		return i.offset == i.offsetLimit
+	}
+	return false
+}
+
+func (i *blockIter) First() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if i.dir == dirBackward {
+		i.prevNode = i.prevNode[:0]
+		i.prevKeys = i.prevKeys[:0]
+	}
+	i.dir = dirSOI
+	return i.Next()
+}
+
+func (i *blockIter) Last() bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if i.dir == dirBackward {
+		i.prevNode = i.prevNode[:0]
+		i.prevKeys = i.prevKeys[:0]
+	}
+	i.dir = dirEOI
+	return i.Prev()
+}
+
+func (i *blockIter) Seek(key []byte) bool {
+	if i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	ri, offset, err := i.block.seek(i.riStart, i.riLimit, key)
+	if err != nil {
+		i.sErr(err)
+		return false
+	}
+	i.restartIndex = ri
+	i.offset = max(i.offsetStart, offset)
+	if i.dir == dirSOI || i.dir == dirEOI {
+		i.dir = dirForward
+	}
+	for i.Next() {
+		if i.block.cmp.Compare(i.key, key) >= 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func (i *blockIter) Next() bool {
+	if i.dir == dirEOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	if i.dir == dirSOI {
+		i.restartIndex = i.riStart
+		i.offset = i.offsetStart
+	} else if i.dir == dirBackward {
+		i.prevNode = i.prevNode[:0]
+		i.prevKeys = i.prevKeys[:0]
+	}
+	for i.offset < i.offsetRealStart {
+		key, value, nShared, n, err := i.block.entry(i.offset)
+		if err != nil {
+			i.sErr(err)
+			return false
+		}
+		if n == 0 {
+			i.dir = dirEOI
+			return false
+		}
+		i.key = append(i.key[:nShared], key...)
+		i.value = value
+		i.offset += n
+	}
+	if i.offset >= i.offsetLimit {
+		i.dir = dirEOI
+		if i.offset != i.offsetLimit {
+			i.sErr(errors.New("leveldb/table: Reader: Next: invalid block (block entries offset not aligned)"))
+		}
+		return false
+	}
+	key, value, nShared, n, err := i.block.entry(i.offset)
+	if err != nil {
+		i.sErr(err)
+		return false
+	}
+	if n == 0 {
+		i.dir = dirEOI
+		return false
+	}
+	i.key = append(i.key[:nShared], key...)
+	i.value = value
+	i.prevOffset = i.offset
+	i.offset += n
+	i.dir = dirForward
+	return true
+}
+
+func (i *blockIter) Prev() bool {
+	if i.dir == dirSOI || i.err != nil {
+		return false
+	} else if i.dir == dirReleased {
+		i.err = ErrIterReleased
+		return false
+	}
+
+	var ri int
+	if i.dir == dirForward {
+		// Change direction.
+		i.offset = i.prevOffset
+		if i.offset == i.offsetRealStart {
+			i.dir = dirSOI
+			return false
+		}
+		ri = i.block.restartIndex(i.restartIndex, i.riLimit, i.offset)
+		i.dir = dirBackward
+	} else if i.dir == dirEOI {
+		// At the end of iterator.
+		i.restartIndex = i.riLimit
+		i.offset = i.offsetLimit
+		if i.offset == i.offsetRealStart {
+			i.dir = dirSOI
+			return false
+		}
+		ri = i.riLimit - 1
+		i.dir = dirBackward
+	} else if len(i.prevNode) == 1 {
+		// This is the end of a restart range.
+		i.offset = i.prevNode[0]
+		i.prevNode = i.prevNode[:0]
+		if i.restartIndex == i.riStart {
+			i.dir = dirSOI
+			return false
+		}
+		i.restartIndex--
+		ri = i.restartIndex
+	} else {
+		// In the middle of restart range, get from cache.
+		n := len(i.prevNode) - 3
+		node := i.prevNode[n:]
+		i.prevNode = i.prevNode[:n]
+		// Get the key.
+		ko := node[0]
+		i.key = append(i.key[:0], i.prevKeys[ko:]...)
+		i.prevKeys = i.prevKeys[:ko]
+		// Get the value.
+		vo := node[1]
+		vl := vo + node[2]
+		i.value = i.block.data[vo:vl]
+		i.offset = vl
+		return true
+	}
+	// Build entries cache.
+	i.key = i.key[:0]
+	i.value = nil
+	offset := i.block.restartOffset(ri)
+	if offset == i.offset {
+		ri -= 1
+		if ri < 0 {
+			i.dir = dirSOI
+			return false
+		}
+		offset = i.block.restartOffset(ri)
+	}
+	i.prevNode = append(i.prevNode, offset)
+	for {
+		key, value, nShared, n, err := i.block.entry(offset)
+		if err != nil {
+			i.sErr(err)
+			return false
+		}
+		if offset >= i.offsetRealStart {
+			if i.value != nil {
+				// Appends 3 variables:
+				// 1. Previous keys offset
+				// 2. Value offset in the data block
+				// 3. Value length
+				i.prevNode = append(i.prevNode, len(i.prevKeys), offset-len(i.value), len(i.value))
+				i.prevKeys = append(i.prevKeys, i.key...)
+			}
+			i.value = value
+		}
+		i.key = append(i.key[:nShared], key...)
+		offset += n
+		// Stop if target offset reached.
+		if offset >= i.offset {
+			if offset != i.offset {
+				i.sErr(errors.New("leveldb/table: Reader: Prev: invalid block (block entries offset not aligned)"))
+				return false
+			}
+
+			break
+		}
+	}
+	i.restartIndex = ri
+	i.offset = offset
+	return true
+}
+
+func (i *blockIter) Key() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.key
+}
+
+func (i *blockIter) Value() []byte {
+	if i.err != nil || i.dir <= dirEOI {
+		return nil
+	}
+	return i.value
+}
+
+func (i *blockIter) Release() {
+	i.prevNode = nil
+	i.prevKeys = nil
+	i.key = nil
+	i.value = nil
+	i.dir = dirReleased
+	if i.cache != nil {
+		i.cache.Release()
+		i.cache = nil
+	}
+	if i.releaser != nil {
+		i.releaser.Release()
+		i.releaser = nil
+	}
+}
+
+func (i *blockIter) SetReleaser(releaser util.Releaser) {
+	if i.dir > dirReleased {
+		i.releaser = releaser
+	}
+}
+
+func (i *blockIter) Valid() bool {
+	return i.err == nil && (i.dir == dirBackward || i.dir == dirForward)
+}
+
+func (i *blockIter) Error() error {
+	return i.err
+}
+
+type filterBlock struct {
+	filter     filter.Filter
+	data       []byte
+	oOffset    int
+	baseLg     uint
+	filtersNum int
+}
+
+func (b *filterBlock) contains(offset uint64, key []byte) bool {
+	i := int(offset >> b.baseLg)
+	if i < b.filtersNum {
+		o := b.data[b.oOffset+i*4:]
+		n := int(binary.LittleEndian.Uint32(o))
+		m := int(binary.LittleEndian.Uint32(o[4:]))
+		if n < m && m <= b.oOffset {
+			return b.filter.Contains(b.data[n:m], key)
+		} else if n == m {
+			return false
+		}
+	}
+	return true
+}
+
+type indexIter struct {
+	blockIter
+	tableReader *Reader
+	slice       *util.Range
+	// Options
+	checksum  bool
+	fillCache bool
+}
+
+func (i *indexIter) Get() iterator.Iterator {
+	value := i.Value()
+	if value == nil {
+		return nil
+	}
+	dataBH, n := decodeBlockHandle(value)
+	if n == 0 {
+		return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)"))
+	}
+	var slice *util.Range
+	if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
+		slice = i.slice
+	}
+	return i.tableReader.getDataIter(dataBH, slice, i.checksum, i.fillCache)
+}
+
+// Reader is a table reader.
+type Reader struct {
+	reader io.ReaderAt
+	cache  cache.Namespace
+	err    error
+	// Options
+	cmp        comparer.Comparer
+	filter     filter.Filter
+	checksum   bool
+	strictIter bool
+
+	dataEnd     int64
+	indexBlock  *block
+	filterBlock *filterBlock
+}
+
+func verifyChecksum(data []byte) bool {
+	n := len(data) - 4
+	checksum0 := binary.LittleEndian.Uint32(data[n:])
+	checksum1 := util.NewCRC(data[:n]).Value()
+	return checksum0 == checksum1
+}
+
+func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
+	data := make([]byte, bh.length+blockTrailerLen)
+	if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
+		return nil, err
+	}
+	if checksum || r.checksum {
+		if !verifyChecksum(data) {
+			return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
+		}
+	}
+	switch data[bh.length] {
+	case blockTypeNoCompression:
+		data = data[:bh.length]
+	case blockTypeSnappyCompression:
+		var err error
+		data, err = snappy.Decode(nil, data[:bh.length])
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length])
+	}
+	return data, nil
+}
+
+func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) {
+	data, err := r.readRawBlock(bh, checksum)
+	if err != nil {
+		return nil, err
+	}
+	restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
+	b := &block{
+		cmp:            r.cmp,
+		data:           data,
+		restartsLen:    restartsLen,
+		restartsOffset: len(data) - (restartsLen+1)*4,
+		checksum:       checksum || r.checksum,
+	}
+	return b, nil
+}
+
+func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterBlock, error) {
+	data, err := r.readRawBlock(bh, true)
+	if err != nil {
+		return nil, err
+	}
+	n := len(data)
+	if n < 5 {
+		return nil, errors.New("leveldb/table: Reader: invalid filter block (too short)")
+	}
+	m := n - 5
+	oOffset := int(binary.LittleEndian.Uint32(data[m:]))
+	if oOffset > m {
+		return nil, errors.New("leveldb/table: Reader: invalid filter block (invalid offset)")
+	}
+	b := &filterBlock{
+		filter:     filter,
+		data:       data,
+		oOffset:    oOffset,
+		baseLg:     uint(data[n-1]),
+		filtersNum: (m - oOffset) / 4,
+	}
+	return b, nil
+}
+
+func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
+	if r.cache != nil {
+		// Get/set block cache.
+		var err error
+		cache, ok := r.cache.Get(dataBH.offset, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+			if !fillCache {
+				return
+			}
+			var dataBlock *block
+			dataBlock, err = r.readBlock(dataBH, checksum)
+			if err == nil {
+				ok = true
+				value = dataBlock
+				charge = int(dataBH.length)
+			}
+			return
+		})
+		if err != nil {
+			return iterator.NewEmptyIterator(err)
+		}
+		if ok {
+			dataBlock := cache.Value().(*block)
+			if !dataBlock.checksum && (r.checksum || checksum) {
+				if !verifyChecksum(dataBlock.data) {
+					return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid block (checksum mismatch)"))
+				}
+				dataBlock.checksum = true
+			}
+			iter := dataBlock.newIterator(slice, false, cache)
+			return iter
+		}
+	}
+	dataBlock, err := r.readBlock(dataBH, checksum)
+	if err != nil {
+		return iterator.NewEmptyIterator(err)
+	}
+	iter := dataBlock.newIterator(slice, false, nil)
+	return iter
+}
+
+// NewIterator creates an iterator from the table.
+//
+// Slice allows slicing the iterator to only contains keys in the given
+// range. A nil Range.Start is treated as a key before all keys in the
+// table. And a nil Range.Limit is treated as a key after all keys in
+// the table.
+//
+// The returned iterator is not goroutine-safe and should be released
+// when not used.
+//
+// Also read Iterator documentation of the leveldb/iterator package.
+
+func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	if r.err != nil {
+		return iterator.NewEmptyIterator(r.err)
+	}
+
+	index := &indexIter{
+		blockIter:   *r.indexBlock.newIterator(slice, true, nil),
+		tableReader: r,
+		slice:       slice,
+		checksum:    ro.GetStrict(opt.StrictBlockChecksum),
+		fillCache:   !ro.GetDontFillCache(),
+	}
+	return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false)
+}
+
+// Find finds key/value pair whose key is greater than or equal to the
+// given key. It returns ErrNotFound if the table doesn't contain
+// such pair.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Find returns.
+func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) {
+	if r.err != nil {
+		err = r.err
+		return
+	}
+
+	index := r.indexBlock.newIterator(nil, true, nil)
+	defer index.Release()
+	if !index.Seek(key) {
+		err = index.Error()
+		if err == nil {
+			err = ErrNotFound
+		}
+		return
+	}
+	dataBH, n := decodeBlockHandle(index.Value())
+	if n == 0 {
+		err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
+		return
+	}
+	if r.filterBlock != nil && !r.filterBlock.contains(dataBH.offset, key) {
+		err = ErrNotFound
+		return
+	}
+	data := r.getDataIter(dataBH, nil, ro.GetStrict(opt.StrictBlockChecksum), !ro.GetDontFillCache())
+	defer data.Release()
+	if !data.Seek(key) {
+		err = data.Error()
+		if err == nil {
+			err = ErrNotFound
+		}
+		return
+	}
+	rkey = data.Key()
+	value = data.Value()
+	return
+}
+
+// Get gets the value for the given key. It returns errors.ErrNotFound
+// if the table does not contain the key.
+//
+// The caller should not modify the contents of the returned slice, but
+// it is safe to modify the contents of the argument after Get returns.
+func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	if r.err != nil {
+		err = r.err
+		return
+	}
+
+	rkey, value, err := r.Find(key, ro)
+	if err == nil && r.cmp.Compare(rkey, key) != 0 {
+		value = nil
+		err = ErrNotFound
+	}
+	return
+}
+
+// GetApproximateOffset returns approximate offset for the given key.
+//
+// It is safe to modify the contents of the argument after Get returns.
+func (r *Reader) GetApproximateOffset(key []byte) (offset int64, err error) {
+	if r.err != nil {
+		err = r.err
+		return
+	}
+
+	index := r.indexBlock.newIterator(nil, true, nil)
+	defer index.Release()
+	if index.Seek(key) {
+		dataBH, n := decodeBlockHandle(index.Value())
+		if n == 0 {
+			err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
+			return
+		}
+		offset = int64(dataBH.offset)
+		return
+	}
+	err = index.Error()
+	if err == nil {
+		offset = r.dataEnd
+	}
+	return
+}
+
+// NewReader creates a new initialized table reader for the file.
+// The cache is optional and can be nil.
+func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options) *Reader {
+	r := &Reader{
+		reader:     f,
+		cache:      cache,
+		cmp:        o.GetComparer(),
+		checksum:   o.GetStrict(opt.StrictBlockChecksum),
+		strictIter: o.GetStrict(opt.StrictIterator),
+	}
+	if f == nil {
+		r.err = errors.New("leveldb/table: Reader: nil file")
+		return r
+	}
+	if size < footerLen {
+		r.err = errors.New("leveldb/table: Reader: invalid table (file size is too small)")
+		return r
+	}
+	var footer [footerLen]byte
+	if _, err := r.reader.ReadAt(footer[:], size-footerLen); err != nil && err != io.EOF {
+		r.err = fmt.Errorf("leveldb/table: Reader: invalid table (could not read footer): %v", err)
+	}
+	if string(footer[footerLen-len(magic):footerLen]) != magic {
+		r.err = errors.New("leveldb/table: Reader: invalid table (bad magic number)")
+		return r
+	}
+	// Decode the metaindex block handle.
+	metaBH, n := decodeBlockHandle(footer[:])
+	if n == 0 {
+		r.err = errors.New("leveldb/table: Reader: invalid table (bad metaindex block handle)")
+		return r
+	}
+	// Decode the index block handle.
+	indexBH, n := decodeBlockHandle(footer[n:])
+	if n == 0 {
+		r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)")
+		return r
+	}
+	// Read index block.
+	r.indexBlock, r.err = r.readBlock(indexBH, true)
+	if r.err != nil {
+		return r
+	}
+	// Read metaindex block.
+	metaBlock, err := r.readBlock(metaBH, true)
+	if err != nil {
+		r.err = err
+		return r
+	}
+	// Set data end.
+	r.dataEnd = int64(metaBH.offset)
+	metaIter := metaBlock.newIterator(nil, false, nil)
+	for metaIter.Next() {
+		key := string(metaIter.Key())
+		if !strings.HasPrefix(key, "filter.") {
+			continue
+		}
+		fn := key[7:]
+		var filter filter.Filter
+		if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn {
+			filter = f0
+		} else {
+			for _, f0 := range o.GetAltFilters() {
+				if f0.Name() == fn {
+					filter = f0
+					break
+				}
+			}
+		}
+		if filter != nil {
+			filterBH, n := decodeBlockHandle(metaIter.Value())
+			if n == 0 {
+				continue
+			}
+			// Update data end.
+			r.dataEnd = int64(filterBH.offset)
+			filterBlock, err := r.readFilterBlock(filterBH, filter)
+			if err != nil {
+				continue
+			}
+			r.filterBlock = filterBlock
+			break
+		}
+	}
+	metaIter.Release()
+	return r
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package table allows read and write sorted key/value.
+package table
+
+import (
+	"encoding/binary"
+)
+
+/*
+Table:
+
+Table is consist of one or more data blocks, an optional filter block
+a metaindex block, an index block and a table footer. Metaindex block
+is a special block used to keep parameters of the table, such as filter
+block name and its block handle. Index block is a special block used to
+keep record of data blocks offset and length, index block use one as
+restart interval. The key used by index block are the last key of preceding
+block, shorter separator of adjacent blocks or shorter successor of the
+last key of the last block. Filter block is an optional block contains
+sequence of filter data generated by a filter generator.
+
+Table data structure:
+                                                         + optional
+                                                        /
+    +--------------+--------------+--------------+------+-------+-----------------+-------------+--------+
+    | data block 1 |      ...     | data block n | filter block | metaindex block | index block | footer |
+    +--------------+--------------+--------------+--------------+-----------------+-------------+--------+
+
+    Each block followed by a 5-bytes trailer contains compression type and checksum.
+
+Table block trailer:
+
+    +---------------------------+-------------------+
+    | compression type (1-byte) | checksum (4-byte) |
+    +---------------------------+-------------------+
+
+    The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression
+    type also included in the checksum.
+
+Table footer:
+
+      +------------------- 40-bytes -------------------+
+     /                                                  \
+    +------------------------+--------------------+------+-----------------+
+    | metaindex block handle / index block handle / ---- | magic (8-bytes) |
+    +------------------------+--------------------+------+-----------------+
+
+    The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/".
+
+NOTE: All fixed-length integer are little-endian.
+*/
+
+/*
+Block:
+
+Block is consist of one or more key/value entries and a block trailer.
+Block entry shares key prefix with its preceding key until a restart
+point reached. A block should contains at least one restart point.
+First restart point are always zero.
+
+Block data structure:
+
+      + restart point                 + restart point (depends on restart interval)
+     /                               /
+    +---------------+---------------+---------------+---------------+---------+
+    | block entry 1 | block entry 2 |      ...      | block entry n | trailer |
+    +---------------+---------------+---------------+---------------+---------+
+
+Key/value entry:
+
+              +---- key len ----+
+             /                   \
+    +-------+---------+-----------+---------+--------------------+--------------+----------------+
+    | shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) |
+    +-----------------+---------------------+--------------------+--------------+----------------+
+
+    Block entry shares key prefix with its preceding key:
+    Conditions:
+        restart_interval=2
+        entry one  : key=deck,value=v1
+        entry two  : key=dock,value=v2
+        entry three: key=duck,value=v3
+    The entries will be encoded as follow:
+
+      + restart point (offset=0)                                                 + restart point (offset=16)
+     /                                                                          /
+    +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
+    |  0  |  4  |  2  |  "deck"  |  "v1"  |  1  |  3  |  2  |  "ock"  |  "v2"  |  0  |  4  |  2  |  "duck"  |  "v3"  |
+    +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
+     \                                   / \                                  / \                                   /
+      +----------- entry one -----------+   +----------- entry two ----------+   +---------- entry three ----------+
+
+    The block trailer will contains two restart points:
+
+    +------------+-----------+--------+
+    |     0      |    16     |   2    |
+    +------------+-----------+---+----+
+     \                      /     \
+      +-- restart points --+       + restart points length
+
+Block trailer:
+
+      +-- 4-bytes --+
+     /               \
+    +-----------------+-----------------+-----------------+------------------------------+
+    | restart point 1 |       ....      | restart point n | restart points len (4-bytes) |
+    +-----------------+-----------------+-----------------+------------------------------+
+
+
+NOTE: All fixed-length integer are little-endian.
+*/
+
+/*
+Filter block:
+
+Filter block consist of one or more filter data and a filter block trailer.
+The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg.
+
+Filter block data structure:
+
+      + offset 1      + offset 2      + offset n      + trailer offset
+     /               /               /               /
+    +---------------+---------------+---------------+---------+
+    | filter data 1 |      ...      | filter data n | trailer |
+    +---------------+---------------+---------------+---------+
+
+Filter block trailer:
+
+      +- 4-bytes -+
+     /             \
+    +---------------+---------------+---------------+-------------------------+------------------+
+    |    offset 1   |      ....     |    offset n   | filter offset (4-bytes) | base Lg (1-byte) |
+    +-------------- +---------------+---------------+-------------------------+------------------+
+
+
+NOTE: All fixed-length integer are little-endian.
+*/
+
+const (
+	blockTrailerLen = 5
+	footerLen       = 48
+
+	magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb"
+
+	// The block type gives the per-block compression format.
+	// These constants are part of the file format and should not be changed.
+	blockTypeNoCompression     = 0
+	blockTypeSnappyCompression = 1
+
+	// Generate new filter every 2KB of data
+	filterBaseLg = 11
+	filterBase   = 1 << filterBaseLg
+)
+
+type blockHandle struct {
+	offset, length uint64
+}
+
+func decodeBlockHandle(src []byte) (blockHandle, int) {
+	offset, n := binary.Uvarint(src)
+	length, m := binary.Uvarint(src[n:])
+	if n == 0 || m == 0 {
+		return blockHandle{}, 0
+	}
+	return blockHandle{offset, length}, n + m
+}
+
+func encodeBlockHandle(dst []byte, b blockHandle) int {
+	n := binary.PutUvarint(dst, b.offset)
+	m := binary.PutUvarint(dst[n:], b.length)
+	return n + m
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go

@@ -0,0 +1,17 @@
+package table
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+)
+
+func TestTable(t *testing.T) {
+	testutil.RunDefer()
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Table Suite")
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go

@@ -0,0 +1,119 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package table
+
+import (
+	"bytes"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type tableWrapper struct {
+	*Reader
+}
+
+func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) {
+	return t.Reader.Find(key, nil)
+}
+
+func (t tableWrapper) TestGet(key []byte) (value []byte, err error) {
+	return t.Reader.Get(key, nil)
+}
+
+func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator {
+	return t.Reader.NewIterator(slice, nil)
+}
+
+var _ = testutil.Defer(func() {
+	Describe("Table", func() {
+		Describe("approximate offset test", func() {
+			var (
+				buf = &bytes.Buffer{}
+				o   = &opt.Options{
+					BlockSize:   1024,
+					Compression: opt.NoCompression,
+				}
+			)
+
+			// Building the table.
+			tw := NewWriter(buf, o)
+			tw.Append([]byte("k01"), []byte("hello"))
+			tw.Append([]byte("k02"), []byte("hello2"))
+			tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000))
+			tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000))
+			tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000))
+			tw.Append([]byte("k06"), []byte("hello3"))
+			tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000))
+			err := tw.Close()
+
+			It("Should be able to approximate offset of a key correctly", func() {
+				Expect(err).ShouldNot(HaveOccurred())
+
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				CheckOffset := func(key string, expect, threshold int) {
+					offset, err := tr.GetApproximateOffset([]byte(key))
+					Expect(err).ShouldNot(HaveOccurred())
+					Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key)
+				}
+
+				CheckOffset("k0", 0, 0)
+				CheckOffset("k01a", 0, 0)
+				CheckOffset("k02", 0, 0)
+				CheckOffset("k03", 0, 0)
+				CheckOffset("k04", 10000, 1000)
+				CheckOffset("k04a", 210000, 1000)
+				CheckOffset("k05", 210000, 1000)
+				CheckOffset("k06", 510000, 1000)
+				CheckOffset("k07", 510000, 1000)
+				CheckOffset("xyz", 610000, 2000)
+			})
+		})
+
+		Describe("read test", func() {
+			Build := func(kv testutil.KeyValue) testutil.DB {
+				o := &opt.Options{
+					BlockSize:            512,
+					BlockRestartInterval: 3,
+				}
+				buf := &bytes.Buffer{}
+
+				// Building the table.
+				tw := NewWriter(buf, o)
+				kv.Iterate(func(i int, key, value []byte) {
+					tw.Append(key, value)
+				})
+				tw.Close()
+
+				// Opening the table.
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				return tableWrapper{tr}
+			}
+			Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
+				return func() {
+					db := Build(*kv)
+					if body != nil {
+						body(db.(tableWrapper).Reader)
+					}
+					testutil.KeyValueTesting(nil, db, *kv)
+				}
+			}
+
+			testutil.AllKeyValueTesting(nil, Build)
+			Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
+				It("should have correct blocks number", func() {
+					Expect(r.indexBlock.restartsLen).Should(Equal(9))
+				})
+			}))
+		})
+	})
+})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go

@@ -0,0 +1,379 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package table
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+
+	"code.google.com/p/snappy-go/snappy"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func sharedPrefixLen(a, b []byte) int {
+	i, n := 0, len(a)
+	if n > len(b) {
+		n = len(b)
+	}
+	for i < n && a[i] == b[i] {
+		i++
+	}
+	return i
+}
+
+type blockWriter struct {
+	restartInterval int
+	buf             util.Buffer
+	nEntries        int
+	prevKey         []byte
+	restarts        []uint32
+	scratch         []byte
+}
+
+func (w *blockWriter) append(key, value []byte) {
+	nShared := 0
+	if w.nEntries%w.restartInterval == 0 {
+		w.restarts = append(w.restarts, uint32(w.buf.Len()))
+	} else {
+		nShared = sharedPrefixLen(w.prevKey, key)
+	}
+	n := binary.PutUvarint(w.scratch[0:], uint64(nShared))
+	n += binary.PutUvarint(w.scratch[n:], uint64(len(key)-nShared))
+	n += binary.PutUvarint(w.scratch[n:], uint64(len(value)))
+	w.buf.Write(w.scratch[:n])
+	w.buf.Write(key[nShared:])
+	w.buf.Write(value)
+	w.prevKey = append(w.prevKey[:0], key...)
+	w.nEntries++
+}
+
+func (w *blockWriter) finish() {
+	// Write restarts entry.
+	if w.nEntries == 0 {
+		// Must have at least one restart entry.
+		w.restarts = append(w.restarts, 0)
+	}
+	w.restarts = append(w.restarts, uint32(len(w.restarts)))
+	for _, x := range w.restarts {
+		buf4 := w.buf.Alloc(4)
+		binary.LittleEndian.PutUint32(buf4, x)
+	}
+}
+
+func (w *blockWriter) reset() {
+	w.buf.Reset()
+	w.nEntries = 0
+	w.restarts = w.restarts[:0]
+}
+
+func (w *blockWriter) bytesLen() int {
+	restartsLen := len(w.restarts)
+	if restartsLen == 0 {
+		restartsLen = 1
+	}
+	return w.buf.Len() + 4*restartsLen + 4
+}
+
+type filterWriter struct {
+	generator filter.FilterGenerator
+	buf       util.Buffer
+	nKeys     int
+	offsets   []uint32
+}
+
+func (w *filterWriter) add(key []byte) {
+	if w.generator == nil {
+		return
+	}
+	w.generator.Add(key)
+	w.nKeys++
+}
+
+func (w *filterWriter) flush(offset uint64) {
+	if w.generator == nil {
+		return
+	}
+	for x := int(offset / filterBase); x > len(w.offsets); {
+		w.generate()
+	}
+}
+
+func (w *filterWriter) finish() {
+	if w.generator == nil {
+		return
+	}
+	// Generate last keys.
+
+	if w.nKeys > 0 {
+		w.generate()
+	}
+	w.offsets = append(w.offsets, uint32(w.buf.Len()))
+	for _, x := range w.offsets {
+		buf4 := w.buf.Alloc(4)
+		binary.LittleEndian.PutUint32(buf4, x)
+	}
+	w.buf.WriteByte(filterBaseLg)
+}
+
+func (w *filterWriter) generate() {
+	// Record offset.
+	w.offsets = append(w.offsets, uint32(w.buf.Len()))
+	// Generate filters.
+	if w.nKeys > 0 {
+		w.generator.Generate(&w.buf)
+		w.nKeys = 0
+	}
+}
+
+// Writer is a table writer.
+type Writer struct {
+	writer io.Writer
+	err    error
+	// Options
+	cmp         comparer.Comparer
+	filter      filter.Filter
+	compression opt.Compression
+	blockSize   int
+
+	dataBlock   blockWriter
+	indexBlock  blockWriter
+	filterBlock filterWriter
+	pendingBH   blockHandle
+	offset      uint64
+	nEntries    int
+	// Scratch allocated enough for 5 uvarint. Block writer should not use
+	// first 20-bytes since it will be used to encode block handle, which
+	// then passed to the block writer itself.
+	scratch            [50]byte
+	comparerScratch    []byte
+	compressionScratch []byte
+}
+
+func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) {
+	// Compress the buffer if necessary.
+	var b []byte
+	if compression == opt.SnappyCompression {
+		// Allocate scratch enough for compression and block trailer.
+		if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
+			w.compressionScratch = make([]byte, n)
+		}
+		var compressed []byte
+		compressed, err = snappy.Encode(w.compressionScratch, buf.Bytes())
+		if err != nil {
+			return
+		}
+		n := len(compressed)
+		b = compressed[:n+blockTrailerLen]
+		b[n] = blockTypeSnappyCompression
+	} else {
+		tmp := buf.Alloc(blockTrailerLen)
+		tmp[0] = blockTypeNoCompression
+		b = buf.Bytes()
+	}
+
+	// Calculate the checksum.
+	n := len(b) - 4
+	checksum := util.NewCRC(b[:n]).Value()
+	binary.LittleEndian.PutUint32(b[n:], checksum)
+
+	// Write the buffer to the file.
+	_, err = w.writer.Write(b)
+	if err != nil {
+		return
+	}
+	bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)}
+	w.offset += uint64(len(b))
+	return
+}
+
+func (w *Writer) flushPendingBH(key []byte) {
+	if w.pendingBH.length == 0 {
+		return
+	}
+	var separator []byte
+	if len(key) == 0 {
+		separator = w.cmp.Successor(w.comparerScratch[:0], w.dataBlock.prevKey)
+	} else {
+		separator = w.cmp.Separator(w.comparerScratch[:0], w.dataBlock.prevKey, key)
+	}
+	if separator == nil {
+		separator = w.dataBlock.prevKey
+	} else {
+		w.comparerScratch = separator
+	}
+	n := encodeBlockHandle(w.scratch[:20], w.pendingBH)
+	// Append the block handle to the index block.
+	w.indexBlock.append(separator, w.scratch[:n])
+	// Reset prev key of the data block.
+	w.dataBlock.prevKey = w.dataBlock.prevKey[:0]
+	// Clear pending block handle.
+	w.pendingBH = blockHandle{}
+}
+
+func (w *Writer) finishBlock() error {
+	w.dataBlock.finish()
+	bh, err := w.writeBlock(&w.dataBlock.buf, w.compression)
+	if err != nil {
+		return err
+	}
+	w.pendingBH = bh
+	// Reset the data block.
+	w.dataBlock.reset()
+	// Flush the filter block.
+	w.filterBlock.flush(w.offset)
+	return nil
+}
+
+// Append appends key/value pair to the table. The keys passed must
+// be in increasing order.
+//
+// It is safe to modify the contents of the arguments after Append returns.
+func (w *Writer) Append(key, value []byte) error {
+	if w.err != nil {
+		return w.err
+	}
+	if w.nEntries > 0 && w.cmp.Compare(w.dataBlock.prevKey, key) >= 0 {
+		w.err = fmt.Errorf("leveldb/table: Writer: keys are not in increasing order: %q, %q", w.dataBlock.prevKey, key)
+		return w.err
+	}
+
+	w.flushPendingBH(key)
+	// Append key/value pair to the data block.
+	w.dataBlock.append(key, value)
+	// Add key to the filter block.
+	w.filterBlock.add(key)
+
+	// Finish the data block if block size target reached.
+	if w.dataBlock.bytesLen() >= w.blockSize {
+		if err := w.finishBlock(); err != nil {
+			w.err = err
+			return w.err
+		}
+	}
+	w.nEntries++
+	return nil
+}
+
+// BlocksLen returns number of blocks written so far.
+func (w *Writer) BlocksLen() int {
+	n := w.indexBlock.nEntries
+	if w.pendingBH.length > 0 {
+		// Includes the pending block.
+		n++
+	}
+	return n
+}
+
+// EntriesLen returns number of entries added so far.
+func (w *Writer) EntriesLen() int {
+	return w.nEntries
+}
+
+// BytesLen returns number of bytes written so far.
+func (w *Writer) BytesLen() int {
+	return int(w.offset)
+}
+
+// Close will finalize the table. Calling Append is not possible
+// after Close, but calling BlocksLen, EntriesLen and BytesLen
+// is still possible.
+func (w *Writer) Close() error {
+	if w.err != nil {
+		return w.err
+	}
+
+	// Write the last data block. Or empty data block if there
+	// aren't any data blocks at all.
+	if w.dataBlock.nEntries > 0 || w.nEntries == 0 {
+		if err := w.finishBlock(); err != nil {
+			w.err = err
+			return w.err
+		}
+	}
+	w.flushPendingBH(nil)
+
+	// Write the filter block.
+	var filterBH blockHandle
+	w.filterBlock.finish()
+	if buf := &w.filterBlock.buf; buf.Len() > 0 {
+		filterBH, w.err = w.writeBlock(buf, opt.NoCompression)
+		if w.err != nil {
+			return w.err
+		}
+	}
+
+	// Write the metaindex block.
+	if filterBH.length > 0 {
+		key := []byte("filter." + w.filter.Name())
+		n := encodeBlockHandle(w.scratch[:20], filterBH)
+		w.dataBlock.append(key, w.scratch[:n])
+	}
+	w.dataBlock.finish()
+	metaindexBH, err := w.writeBlock(&w.dataBlock.buf, w.compression)
+	if err != nil {
+		w.err = err
+		return w.err
+	}
+
+	// Write the index block.
+	w.indexBlock.finish()
+	indexBH, err := w.writeBlock(&w.indexBlock.buf, w.compression)
+	if err != nil {
+		w.err = err
+		return w.err
+	}
+
+	// Write the table footer.
+	footer := w.scratch[:footerLen]
+	for i := range footer {
+		footer[i] = 0
+	}
+	n := encodeBlockHandle(footer, metaindexBH)
+	encodeBlockHandle(footer[n:], indexBH)
+	copy(footer[footerLen-len(magic):], magic)
+	if _, err := w.writer.Write(footer); err != nil {
+		w.err = err
+		return w.err
+	}
+	w.offset += footerLen
+
+	w.err = errors.New("leveldb/table: writer is closed")
+	return nil
+}
+
+// NewWriter creates a new initialized table writer for the file.
+//
+// Table writer is not goroutine-safe.
+func NewWriter(f io.Writer, o *opt.Options) *Writer {
+	w := &Writer{
+		writer:          f,
+		cmp:             o.GetComparer(),
+		filter:          o.GetFilter(),
+		compression:     o.GetCompression(),
+		blockSize:       o.GetBlockSize(),
+		comparerScratch: make([]byte, 0),
+	}
+	// data block
+	w.dataBlock.restartInterval = o.GetBlockRestartInterval()
+	// The first 20-bytes are used for encoding block handle.
+	w.dataBlock.scratch = w.scratch[20:]
+	// index block
+	w.indexBlock.restartInterval = 1
+	w.indexBlock.scratch = w.scratch[20:]
+	// filter block
+	if w.filter != nil {
+		w.filterBlock.generator = w.filter.NewGenerator()
+		w.filterBlock.flush(0)
+	}
+	return w
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type DB interface{}
+
+type Put interface {
+	TestPut(key []byte, value []byte) error
+}
+
+type Delete interface {
+	TestDelete(key []byte) error
+}
+
+type Find interface {
+	TestFind(key []byte) (rkey, rvalue []byte, err error)
+}
+
+type Get interface {
+	TestGet(key []byte) (value []byte, err error)
+}
+
+type NewIterator interface {
+	TestNewIterator(slice *util.Range) iterator.Iterator
+}
+
+type DBAct int
+
+func (a DBAct) String() string {
+	switch a {
+	case DBNone:
+		return "none"
+	case DBPut:
+		return "put"
+	case DBOverwrite:
+		return "overwrite"
+	case DBDelete:
+		return "delete"
+	case DBDeleteNA:
+		return "delete_na"
+	}
+	return "unknown"
+}
+
+const (
+	DBNone DBAct = iota
+	DBPut
+	DBOverwrite
+	DBDelete
+	DBDeleteNA
+)
+
+type DBTesting struct {
+	Rand *rand.Rand
+	DB   interface {
+		Get
+		Put
+		Delete
+	}
+	PostFn             func(t *DBTesting)
+	Deleted, Present   KeyValue
+	Act, LastAct       DBAct
+	ActKey, LastActKey []byte
+}
+
+func (t *DBTesting) post() {
+	if t.PostFn != nil {
+		t.PostFn(t)
+	}
+}
+
+func (t *DBTesting) setAct(act DBAct, key []byte) {
+	t.LastAct, t.Act = t.Act, act
+	t.LastActKey, t.ActKey = t.ActKey, key
+}
+
+func (t *DBTesting) text() string {
+	return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey)
+}
+
+func (t *DBTesting) Text() string {
+	return "DBTesting " + t.text()
+}
+
+func (t *DBTesting) TestPresentKV(key, value []byte) {
+	rvalue, err := t.DB.TestGet(key)
+	Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text())
+	Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text())
+}
+
+func (t *DBTesting) TestAllPresent() {
+	t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) {
+		t.TestPresentKV(key, value)
+	})
+}
+
+func (t *DBTesting) TestDeletedKey(key []byte) {
+	_, err := t.DB.TestGet(key)
+	Expect(err).Should(Equal(util.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
+}
+
+func (t *DBTesting) TestAllDeleted() {
+	t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) {
+		t.TestDeletedKey(key)
+	})
+}
+
+func (t *DBTesting) TestAll() {
+	dn := t.Deleted.Len()
+	pn := t.Present.Len()
+	ShuffledIndex(t.Rand, dn+pn, 1, func(i int) {
+		if i >= dn {
+			key, value := t.Present.Index(i - dn)
+			t.TestPresentKV(key, value)
+		} else {
+			t.TestDeletedKey(t.Deleted.KeyAt(i))
+		}
+	})
+}
+
+func (t *DBTesting) Put(key, value []byte) {
+	if new := t.Present.PutU(key, value); new {
+		t.setAct(DBPut, key)
+	} else {
+		t.setAct(DBOverwrite, key)
+	}
+	t.Deleted.Delete(key)
+	err := t.DB.TestPut(key, value)
+	Expect(err).ShouldNot(HaveOccurred(), t.Text())
+	t.TestPresentKV(key, value)
+	t.post()
+}
+
+func (t *DBTesting) PutRandom() bool {
+	if t.Deleted.Len() > 0 {
+		i := t.Rand.Intn(t.Deleted.Len())
+		key, value := t.Deleted.Index(i)
+		t.Put(key, value)
+		return true
+	}
+	return false
+}
+
+func (t *DBTesting) Delete(key []byte) {
+	if exist, value := t.Present.Delete(key); exist {
+		t.setAct(DBDelete, key)
+		t.Deleted.PutU(key, value)
+	} else {
+		t.setAct(DBDeleteNA, key)
+	}
+	err := t.DB.TestDelete(key)
+	Expect(err).ShouldNot(HaveOccurred(), t.Text())
+	t.TestDeletedKey(key)
+	t.post()
+}
+
+func (t *DBTesting) DeleteRandom() bool {
+	if t.Present.Len() > 0 {
+		i := t.Rand.Intn(t.Present.Len())
+		t.Delete(t.Present.KeyAt(i))
+		return true
+	}
+	return false
+}
+
+func (t *DBTesting) RandomAct(round int) {
+	for i := 0; i < round; i++ {
+		if t.Rand.Int()%2 == 0 {
+			t.PutRandom()
+		} else {
+			t.DeleteRandom()
+		}
+	}
+}
+
+func DoDBTesting(t *DBTesting) {
+	if t.Rand == nil {
+		t.Rand = NewRand()
+	}
+
+	t.DeleteRandom()
+	t.PutRandom()
+	t.DeleteRandom()
+	t.DeleteRandom()
+	for i := t.Deleted.Len() / 2; i >= 0; i-- {
+		t.PutRandom()
+	}
+	t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10)
+
+	// Additional iterator testing
+	if db, ok := t.DB.(NewIterator); ok {
+		iter := db.TestNewIterator(nil)
+		Expect(iter.Error()).NotTo(HaveOccurred())
+
+		it := IteratorTesting{
+			KeyValue: t.Present,
+			Iter:     iter,
+		}
+
+		DoIteratorTesting(&it)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/iter.go

@@ -0,0 +1,327 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+)
+
+type IterAct int
+
+func (a IterAct) String() string {
+	switch a {
+	case IterNone:
+		return "none"
+	case IterFirst:
+		return "first"
+	case IterLast:
+		return "last"
+	case IterPrev:
+		return "prev"
+	case IterNext:
+		return "next"
+	case IterSeek:
+		return "seek"
+	case IterSOI:
+		return "soi"
+	case IterEOI:
+		return "eoi"
+	}
+	return "unknown"
+}
+
+const (
+	IterNone IterAct = iota
+	IterFirst
+	IterLast
+	IterPrev
+	IterNext
+	IterSeek
+	IterSOI
+	IterEOI
+)
+
+type IteratorTesting struct {
+	KeyValue
+	Iter         iterator.Iterator
+	Rand         *rand.Rand
+	PostFn       func(t *IteratorTesting)
+	Pos          int
+	Act, LastAct IterAct
+
+	once bool
+}
+
+func (t *IteratorTesting) init() {
+	if !t.once {
+		t.Pos = -1
+		t.once = true
+	}
+}
+
+func (t *IteratorTesting) post() {
+	if t.PostFn != nil {
+		t.PostFn(t)
+	}
+}
+
+func (t *IteratorTesting) setAct(act IterAct) {
+	t.LastAct, t.Act = t.Act, act
+}
+
+func (t *IteratorTesting) text() string {
+	return fmt.Sprintf("at pos %d and last action was <%v> -> <%v>", t.Pos, t.LastAct, t.Act)
+}
+
+func (t *IteratorTesting) Text() string {
+	return "IteratorTesting is " + t.text()
+}
+
+func (t *IteratorTesting) IsFirst() bool {
+	t.init()
+	return t.Len() > 0 && t.Pos == 0
+}
+
+func (t *IteratorTesting) IsLast() bool {
+	t.init()
+	return t.Len() > 0 && t.Pos == t.Len()-1
+}
+
+func (t *IteratorTesting) TestKV() {
+	t.init()
+	key, value := t.Index(t.Pos)
+	Expect(t.Iter.Key()).NotTo(BeNil())
+	Expect(t.Iter.Key()).Should(Equal(key), "Key is invalid, %s", t.text())
+	Expect(t.Iter.Value()).Should(Equal(value), "Value for key %q, %s", key, t.text())
+}
+
+func (t *IteratorTesting) First() {
+	t.init()
+	t.setAct(IterFirst)
+
+	ok := t.Iter.First()
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	if t.Len() > 0 {
+		t.Pos = 0
+		Expect(ok).Should(BeTrue(), t.Text())
+		t.TestKV()
+	} else {
+		t.Pos = -1
+		Expect(ok).ShouldNot(BeTrue(), t.Text())
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) Last() {
+	t.init()
+	t.setAct(IterLast)
+
+	ok := t.Iter.Last()
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	if t.Len() > 0 {
+		t.Pos = t.Len() - 1
+		Expect(ok).Should(BeTrue(), t.Text())
+		t.TestKV()
+	} else {
+		t.Pos = 0
+		Expect(ok).ShouldNot(BeTrue(), t.Text())
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) Next() {
+	t.init()
+	t.setAct(IterNext)
+
+	ok := t.Iter.Next()
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	if t.Pos < t.Len()-1 {
+		t.Pos++
+		Expect(ok).Should(BeTrue(), t.Text())
+		t.TestKV()
+	} else {
+		t.Pos = t.Len()
+		Expect(ok).ShouldNot(BeTrue(), t.Text())
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) Prev() {
+	t.init()
+	t.setAct(IterPrev)
+
+	ok := t.Iter.Prev()
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	if t.Pos > 0 {
+		t.Pos--
+		Expect(ok).Should(BeTrue(), t.Text())
+		t.TestKV()
+	} else {
+		t.Pos = -1
+		Expect(ok).ShouldNot(BeTrue(), t.Text())
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) Seek(i int) {
+	t.init()
+	t.setAct(IterSeek)
+
+	key, _ := t.Index(i)
+	oldKey, _ := t.IndexOrNil(t.Pos)
+
+	ok := t.Iter.Seek(key)
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q, to pos %d, %s", oldKey, key, i, t.text()))
+
+	t.Pos = i
+	t.TestKV()
+	t.post()
+}
+
+func (t *IteratorTesting) SeekInexact(i int) {
+	t.init()
+	t.setAct(IterSeek)
+	var key0 []byte
+	key1, _ := t.Index(i)
+	if i > 0 {
+		key0, _ = t.Index(i - 1)
+	}
+	key := BytesSeparator(key0, key1)
+	oldKey, _ := t.IndexOrNil(t.Pos)
+
+	ok := t.Iter.Seek(key)
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key1, i, t.text()))
+
+	t.Pos = i
+	t.TestKV()
+	t.post()
+}
+
+func (t *IteratorTesting) SeekKey(key []byte) {
+	t.init()
+	t.setAct(IterSeek)
+	oldKey, _ := t.IndexOrNil(t.Pos)
+	i := t.Search(key)
+
+	ok := t.Iter.Seek(key)
+	Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
+	if i < t.Len() {
+		key_, _ := t.Index(i)
+		Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key_, i, t.text()))
+		t.Pos = i
+		t.TestKV()
+	} else {
+		Expect(ok).ShouldNot(BeTrue(), fmt.Sprintf("Seek from key %q to %q, %s", oldKey, key, t.text()))
+	}
+
+	t.Pos = i
+	t.post()
+}
+
+func (t *IteratorTesting) SOI() {
+	t.init()
+	t.setAct(IterSOI)
+	Expect(t.Pos).Should(BeNumerically("<=", 0), t.Text())
+	for i := 0; i < 3; i++ {
+		t.Prev()
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) EOI() {
+	t.init()
+	t.setAct(IterEOI)
+	Expect(t.Pos).Should(BeNumerically(">=", t.Len()-1), t.Text())
+	for i := 0; i < 3; i++ {
+		t.Next()
+	}
+	t.post()
+}
+
+func (t *IteratorTesting) WalkPrev(fn func(t *IteratorTesting)) {
+	t.init()
+	for old := t.Pos; t.Pos > 0; old = t.Pos {
+		fn(t)
+		Expect(t.Pos).Should(BeNumerically("<", old), t.Text())
+	}
+}
+
+func (t *IteratorTesting) WalkNext(fn func(t *IteratorTesting)) {
+	t.init()
+	for old := t.Pos; t.Pos < t.Len()-1; old = t.Pos {
+		fn(t)
+		Expect(t.Pos).Should(BeNumerically(">", old), t.Text())
+	}
+}
+
+func (t *IteratorTesting) PrevAll() {
+	t.WalkPrev(func(t *IteratorTesting) {
+		t.Prev()
+	})
+}
+
+func (t *IteratorTesting) NextAll() {
+	t.WalkNext(func(t *IteratorTesting) {
+		t.Next()
+	})
+}
+
+func DoIteratorTesting(t *IteratorTesting) {
+	if t.Rand == nil {
+		t.Rand = NewRand()
+	}
+	t.SOI()
+	t.NextAll()
+	t.First()
+	t.SOI()
+	t.NextAll()
+	t.EOI()
+	t.PrevAll()
+	t.Last()
+	t.EOI()
+	t.PrevAll()
+	t.SOI()
+
+	t.NextAll()
+	t.PrevAll()
+	t.NextAll()
+	t.Last()
+	t.PrevAll()
+	t.First()
+	t.NextAll()
+	t.EOI()
+
+	ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
+		t.Seek(i)
+	})
+
+	ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
+		t.SeekInexact(i)
+	})
+
+	ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
+		t.Seek(i)
+		if i%2 != 0 {
+			t.PrevAll()
+			t.SOI()
+		} else {
+			t.NextAll()
+			t.EOI()
+		}
+	})
+
+	for _, key := range []string{"", "foo", "bar", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"} {
+		t.SeekKey([]byte(key))
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kv.go

@@ -0,0 +1,352 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type KeyValueEntry struct {
+	key, value []byte
+}
+
+type KeyValue struct {
+	entries []KeyValueEntry
+	nbytes  int
+}
+
+func (kv *KeyValue) Put(key, value []byte) {
+	if n := len(kv.entries); n > 0 && cmp.Compare(kv.entries[n-1].key, key) >= 0 {
+		panic(fmt.Sprintf("Put: keys are not in increasing order: %q, %q", kv.entries[n-1].key, key))
+	}
+	kv.entries = append(kv.entries, KeyValueEntry{key, value})
+	kv.nbytes += len(key) + len(value)
+}
+
+func (kv *KeyValue) PutString(key, value string) {
+	kv.Put([]byte(key), []byte(value))
+}
+
+func (kv *KeyValue) PutU(key, value []byte) bool {
+	if i, exist := kv.Get(key); !exist {
+		if i < kv.Len() {
+			kv.entries = append(kv.entries[:i+1], kv.entries[i:]...)
+			kv.entries[i] = KeyValueEntry{key, value}
+		} else {
+			kv.entries = append(kv.entries, KeyValueEntry{key, value})
+		}
+		kv.nbytes += len(key) + len(value)
+		return true
+	} else {
+		kv.nbytes += len(value) - len(kv.ValueAt(i))
+		kv.entries[i].value = value
+	}
+	return false
+}
+
+func (kv *KeyValue) PutUString(key, value string) bool {
+	return kv.PutU([]byte(key), []byte(value))
+}
+
+func (kv *KeyValue) Delete(key []byte) (exist bool, value []byte) {
+	i, exist := kv.Get(key)
+	if exist {
+		value = kv.entries[i].value
+		kv.DeleteIndex(i)
+	}
+	return
+}
+
+func (kv *KeyValue) DeleteIndex(i int) bool {
+	if i < kv.Len() {
+		kv.nbytes -= len(kv.KeyAt(i)) + len(kv.ValueAt(i))
+		kv.entries = append(kv.entries[:i], kv.entries[i+1:]...)
+		return true
+	}
+	return false
+}
+
+func (kv KeyValue) Len() int {
+	return len(kv.entries)
+}
+
+func (kv *KeyValue) Size() int {
+	return kv.nbytes
+}
+
+func (kv KeyValue) KeyAt(i int) []byte {
+	return kv.entries[i].key
+}
+
+func (kv KeyValue) ValueAt(i int) []byte {
+	return kv.entries[i].value
+}
+
+func (kv KeyValue) Index(i int) (key, value []byte) {
+	if i < 0 || i >= len(kv.entries) {
+		panic(fmt.Sprintf("Index #%d: out of range", i))
+	}
+	return kv.entries[i].key, kv.entries[i].value
+}
+
+func (kv KeyValue) IndexInexact(i int) (key_, key, value []byte) {
+	key, value = kv.Index(i)
+	var key0 []byte
+	var key1 = kv.KeyAt(i)
+	if i > 0 {
+		key0 = kv.KeyAt(i - 1)
+	}
+	key_ = BytesSeparator(key0, key1)
+	return
+}
+
+func (kv KeyValue) IndexOrNil(i int) (key, value []byte) {
+	if i >= 0 && i < len(kv.entries) {
+		return kv.entries[i].key, kv.entries[i].value
+	}
+	return nil, nil
+}
+
+func (kv KeyValue) IndexString(i int) (key, value string) {
+	key_, _value := kv.Index(i)
+	return string(key_), string(_value)
+}
+
+func (kv KeyValue) Search(key []byte) int {
+	return sort.Search(kv.Len(), func(i int) bool {
+		return cmp.Compare(kv.KeyAt(i), key) >= 0
+	})
+}
+
+func (kv KeyValue) SearchString(key string) int {
+	return kv.Search([]byte(key))
+}
+
+func (kv KeyValue) Get(key []byte) (i int, exist bool) {
+	i = kv.Search(key)
+	if i < kv.Len() && cmp.Compare(kv.KeyAt(i), key) == 0 {
+		exist = true
+	}
+	return
+}
+
+func (kv KeyValue) GetString(key string) (i int, exist bool) {
+	return kv.Get([]byte(key))
+}
+
+func (kv KeyValue) Iterate(fn func(i int, key, value []byte)) {
+	for i, x := range kv.entries {
+		fn(i, x.key, x.value)
+	}
+}
+
+func (kv KeyValue) IterateString(fn func(i int, key, value string)) {
+	kv.Iterate(func(i int, key, value []byte) {
+		fn(i, string(key), string(value))
+	})
+}
+
+func (kv KeyValue) IterateShuffled(rnd *rand.Rand, fn func(i int, key, value []byte)) {
+	ShuffledIndex(rnd, kv.Len(), 1, func(i int) {
+		fn(i, kv.entries[i].key, kv.entries[i].value)
+	})
+}
+
+func (kv KeyValue) IterateShuffledString(rnd *rand.Rand, fn func(i int, key, value string)) {
+	kv.IterateShuffled(rnd, func(i int, key, value []byte) {
+		fn(i, string(key), string(value))
+	})
+}
+
+func (kv KeyValue) IterateInexact(fn func(i int, key_, key, value []byte)) {
+	for i := range kv.entries {
+		key_, key, value := kv.IndexInexact(i)
+		fn(i, key_, key, value)
+	}
+}
+
+func (kv KeyValue) IterateInexactString(fn func(i int, key_, key, value string)) {
+	kv.IterateInexact(func(i int, key_, key, value []byte) {
+		fn(i, string(key_), string(key), string(value))
+	})
+}
+
+func (kv KeyValue) Clone() KeyValue {
+	return KeyValue{append([]KeyValueEntry{}, kv.entries...), kv.nbytes}
+}
+
+func (kv KeyValue) Slice(start, limit int) KeyValue {
+	if start < 0 || limit > kv.Len() {
+		panic(fmt.Sprintf("Slice %d .. %d: out of range", start, limit))
+	} else if limit < start {
+		panic(fmt.Sprintf("Slice %d .. %d: invalid range", start, limit))
+	}
+	return KeyValue{append([]KeyValueEntry{}, kv.entries[start:limit]...), kv.nbytes}
+}
+
+func (kv KeyValue) SliceKey(start, limit []byte) KeyValue {
+	start_ := 0
+	limit_ := kv.Len()
+	if start != nil {
+		start_ = kv.Search(start)
+	}
+	if limit != nil {
+		limit_ = kv.Search(limit)
+	}
+	return kv.Slice(start_, limit_)
+}
+
+func (kv KeyValue) SliceKeyString(start, limit string) KeyValue {
+	return kv.SliceKey([]byte(start), []byte(limit))
+}
+
+func (kv KeyValue) SliceRange(r *util.Range) KeyValue {
+	if r != nil {
+		return kv.SliceKey(r.Start, r.Limit)
+	}
+	return kv.Clone()
+}
+
+func (kv KeyValue) Range(start, limit int) (r util.Range) {
+	if kv.Len() > 0 {
+		if start == kv.Len() {
+			r.Start = BytesAfter(kv.KeyAt(start - 1))
+		} else {
+			r.Start = kv.KeyAt(start)
+		}
+	}
+	if limit < kv.Len() {
+		r.Limit = kv.KeyAt(limit)
+	}
+	return
+}
+
+func KeyValue_EmptyKey() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("", "v")
+	return kv
+}
+
+func KeyValue_EmptyValue() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("abc", "")
+	kv.PutString("abcd", "")
+	return kv
+}
+
+func KeyValue_OneKeyValue() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("abc", "v")
+	return kv
+}
+
+func KeyValue_BigValue() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("big1", strings.Repeat("1", 200000))
+	return kv
+}
+
+func KeyValue_SpecialKey() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("\xff\xff", "v3")
+	return kv
+}
+
+func KeyValue_MultipleKeyValue() *KeyValue {
+	kv := &KeyValue{}
+	kv.PutString("a", "v")
+	kv.PutString("aa", "v1")
+	kv.PutString("aaa", "v2")
+	kv.PutString("aaacccccccccc", "v2")
+	kv.PutString("aaaccccccccccd", "v3")
+	kv.PutString("aaaccccccccccf", "v4")
+	kv.PutString("aaaccccccccccfg", "v5")
+	kv.PutString("ab", "v6")
+	kv.PutString("abc", "v7")
+	kv.PutString("abcd", "v8")
+	kv.PutString("accccccccccccccc", "v9")
+	kv.PutString("b", "v10")
+	kv.PutString("bb", "v11")
+	kv.PutString("bc", "v12")
+	kv.PutString("c", "v13")
+	kv.PutString("c1", "v13")
+	kv.PutString("czzzzzzzzzzzzzz", "v14")
+	kv.PutString("fffffffffffffff", "v15")
+	kv.PutString("g11", "v15")
+	kv.PutString("g111", "v15")
+	kv.PutString("g111\xff", "v15")
+	kv.PutString("zz", "v16")
+	kv.PutString("zzzzzzz", "v16")
+	kv.PutString("zzzzzzzzzzzzzzzz", "v16")
+	return kv
+}
+
+var keymap = []byte("012345678ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy")
+
+func KeyValue_Generate(rnd *rand.Rand, n, minlen, maxlen, vminlen, vmaxlen int) *KeyValue {
+	if rnd == nil {
+		rnd = NewRand()
+	}
+	if maxlen < minlen {
+		panic("max len should >= min len")
+	}
+
+	rrand := func(min, max int) int {
+		if min == max {
+			return max
+		}
+		return rnd.Intn(max-min) + min
+	}
+
+	kv := &KeyValue{}
+	endC := byte(len(keymap) - 1)
+	gen := make([]byte, 0, maxlen)
+	for i := 0; i < n; i++ {
+		m := rrand(minlen, maxlen)
+		last := gen
+	retry:
+		gen = last[:m]
+		if k := len(last); m > k {
+			for j := k; j < m; j++ {
+				gen[j] = 0
+			}
+		} else {
+			for j := m - 1; j >= 0; j-- {
+				c := last[j]
+				if c == endC {
+					continue
+				}
+				gen[j] = c + 1
+				for j += 1; j < m; j++ {
+					gen[j] = 0
+				}
+				goto ok
+			}
+			if m < maxlen {
+				m++
+				goto retry
+			}
+			panic(fmt.Sprintf("only able to generate %d keys out of %d keys, try increasing max len", kv.Len(), n))
+		ok:
+		}
+		key := make([]byte, m)
+		for j := 0; j < m; j++ {
+			key[j] = keymap[gen[j]]
+		}
+		value := make([]byte, rrand(vminlen, vmaxlen))
+		for n := copy(value, []byte(fmt.Sprintf("v%d", i))); n < len(value); n++ {
+			value[n] = 'x'
+		}
+		kv.Put(key, value)
+	}
+	return kv
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
+	if rnd == nil {
+		rnd = NewRand()
+	}
+
+	if db, ok := p.(Find); ok {
+		It("Should find all keys with Find", func() {
+			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
+				key_, key, value := kv.IndexInexact(i)
+
+				// Using exact key.
+				rkey, rvalue, err := db.TestFind(key)
+				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
+				Expect(rkey).Should(Equal(key), "Key")
+				Expect(rvalue).Should(Equal(value), "Value for key %q", key)
+
+				// Using inexact key.
+				rkey, rvalue, err = db.TestFind(key_)
+				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q)", key_, key)
+				Expect(rkey).Should(Equal(key))
+				Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
+			})
+		})
+
+		It("Should return error if the key is not present", func() {
+			var key []byte
+			if kv.Len() > 0 {
+				key_, _ := kv.Index(kv.Len() - 1)
+				key = BytesAfter(key_)
+			}
+			rkey, _, err := db.TestFind(key)
+			Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
+			Expect(err).Should(Equal(util.ErrNotFound))
+		})
+	}
+
+	if db, ok := p.(Get); ok {
+		It("Should only find exact key with Get", func() {
+			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
+				key_, key, value := kv.IndexInexact(i)
+
+				// Using exact key.
+				rvalue, err := db.TestGet(key)
+				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
+				Expect(rvalue).Should(Equal(value), "Value for key %q", key)
+
+				// Using inexact key.
+				if len(key_) > 0 {
+					_, err = db.TestGet(key_)
+					Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
+					Expect(err).Should(Equal(util.ErrNotFound))
+				}
+			})
+		})
+	}
+
+	if db, ok := p.(NewIterator); ok {
+		TestIter := func(r *util.Range, _kv KeyValue) {
+			iter := db.TestNewIterator(r)
+			Expect(iter.Error()).ShouldNot(HaveOccurred())
+
+			t := IteratorTesting{
+				KeyValue: _kv,
+				Iter:     iter,
+			}
+
+			DoIteratorTesting(&t)
+		}
+
+		It("Should iterates and seeks correctly", func(done Done) {
+			TestIter(nil, kv.Clone())
+			done <- true
+		}, 3.0)
+
+		RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
+			type slice struct {
+				r            *util.Range
+				start, limit int
+			}
+
+			key_, _, _ := kv.IndexInexact(i)
+			for _, x := range []slice{
+				{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
+				{&util.Range{Start: nil, Limit: key_}, 0, i},
+			} {
+				It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
+					TestIter(x.r, kv.Slice(x.start, x.limit))
+					done <- true
+				}, 3.0)
+			}
+		})
+
+		RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
+			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
+				r := kv.Range(start, limit)
+				TestIter(&r, kv.Slice(start, limit))
+				done <- true
+			}, 3.0)
+		})
+	}
+}
+
+func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
+	Test := func(kv *KeyValue) func() {
+		return func() {
+			db := body(*kv)
+			KeyValueTesting(rnd, db, *kv)
+		}
+	}
+
+	Describe("with no key/value (empty)", Test(&KeyValue{}))
+	Describe("with empty key", Test(KeyValue_EmptyKey()))
+	Describe("with empty value", Test(KeyValue_EmptyValue()))
+	Describe("with one key/value", Test(KeyValue_OneKeyValue()))
+	Describe("with big value", Test(KeyValue_BigValue()))
+	Describe("with special key", Test(KeyValue_SpecialKey()))
+	Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go

@@ -0,0 +1,585 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	storageMu     sync.Mutex
+	storageUseFS  bool = true
+	storageKeepFS bool = false
+	storageNum    int
+)
+
+type StorageMode int
+
+const (
+	ModeOpen StorageMode = 1 << iota
+	ModeCreate
+	ModeRemove
+	ModeRead
+	ModeWrite
+	ModeSync
+	ModeClose
+)
+
+const (
+	modeOpen = iota
+	modeCreate
+	modeRemove
+	modeRead
+	modeWrite
+	modeSync
+	modeClose
+
+	modeCount
+)
+
+const (
+	typeManifest = iota
+	typeJournal
+	typeTable
+	typeTemp
+
+	typeCount
+)
+
+const flattenCount = modeCount * typeCount
+
+func flattenType(m StorageMode, t storage.FileType) int {
+	var x int
+	switch m {
+	case ModeOpen:
+		x = modeOpen
+	case ModeCreate:
+		x = modeCreate
+	case ModeRemove:
+		x = modeRemove
+	case ModeRead:
+		x = modeRead
+	case ModeWrite:
+		x = modeWrite
+	case ModeSync:
+		x = modeSync
+	case ModeClose:
+		x = modeClose
+	default:
+		panic("invalid storage mode")
+	}
+	x *= typeCount
+	switch t {
+	case storage.TypeManifest:
+		return x + typeManifest
+	case storage.TypeJournal:
+		return x + typeJournal
+	case storage.TypeTable:
+		return x + typeTable
+	case storage.TypeTemp:
+		return x + typeTemp
+	default:
+		panic("invalid file type")
+	}
+}
+
+func listFlattenType(m StorageMode, t storage.FileType) []int {
+	ret := make([]int, 0, flattenCount)
+	add := func(x int) {
+		x *= typeCount
+		switch {
+		case t&storage.TypeManifest != 0:
+			ret = append(ret, x+typeManifest)
+		case t&storage.TypeJournal != 0:
+			ret = append(ret, x+typeJournal)
+		case t&storage.TypeTable != 0:
+			ret = append(ret, x+typeTable)
+		case t&storage.TypeTemp != 0:
+			ret = append(ret, x+typeTemp)
+		}
+	}
+	switch {
+	case m&ModeOpen != 0:
+		add(modeOpen)
+	case m&ModeCreate != 0:
+		add(modeCreate)
+	case m&ModeRemove != 0:
+		add(modeRemove)
+	case m&ModeRead != 0:
+		add(modeRead)
+	case m&ModeWrite != 0:
+		add(modeWrite)
+	case m&ModeSync != 0:
+		add(modeSync)
+	case m&ModeClose != 0:
+		add(modeClose)
+	}
+	return ret
+}
+
+func packFile(num uint64, t storage.FileType) uint64 {
+	if num>>(64-typeCount) != 0 {
+		panic("overflow")
+	}
+	return num<<typeCount | uint64(t)
+}
+
+func unpackFile(x uint64) (uint64, storage.FileType) {
+	return x >> typeCount, storage.FileType(x) & storage.TypeAll
+}
+
+type emulatedError struct {
+	err error
+}
+
+func (err emulatedError) Error() string {
+	return fmt.Sprintf("emulated storage error: %v", err.err)
+}
+
+type storageLock struct {
+	s *Storage
+	r util.Releaser
+}
+
+func (l storageLock) Release() {
+	l.r.Release()
+	l.s.logI("storage lock released")
+}
+
+type reader struct {
+	f *file
+	storage.Reader
+}
+
+func (r *reader) Read(p []byte) (n int, err error) {
+	err = r.f.s.emulateError(ModeRead, r.f.Type())
+	if err == nil {
+		r.f.s.stall(ModeRead, r.f.Type())
+		n, err = r.Reader.Read(p)
+	}
+	r.f.s.count(ModeRead, r.f.Type(), n)
+	if err != nil && err != io.EOF {
+		r.f.s.logI("read error, num=%d type=%v n=%d err=%v", r.f.Num(), r.f.Type(), n, err)
+	}
+	return
+}
+
+func (r *reader) ReadAt(p []byte, off int64) (n int, err error) {
+	err = r.f.s.emulateError(ModeRead, r.f.Type())
+	if err == nil {
+		r.f.s.stall(ModeRead, r.f.Type())
+		n, err = r.Reader.ReadAt(p, off)
+	}
+	r.f.s.count(ModeRead, r.f.Type(), n)
+	if err != nil && err != io.EOF {
+		r.f.s.logI("readAt error, num=%d type=%v offset=%d n=%d err=%v", r.f.Num(), r.f.Type(), off, n, err)
+	}
+	return
+}
+
+func (r *reader) Close() (err error) {
+	return r.f.doClose(r.Reader)
+}
+
+type writer struct {
+	f *file
+	storage.Writer
+}
+
+func (w *writer) Write(p []byte) (n int, err error) {
+	err = w.f.s.emulateError(ModeWrite, w.f.Type())
+	if err == nil {
+		w.f.s.stall(ModeWrite, w.f.Type())
+		n, err = w.Writer.Write(p)
+	}
+	w.f.s.count(ModeWrite, w.f.Type(), n)
+	if err != nil && err != io.EOF {
+		w.f.s.logI("write error, num=%d type=%v n=%d err=%v", w.f.Num(), w.f.Type(), n, err)
+	}
+	return
+}
+
+func (w *writer) Sync() (err error) {
+	err = w.f.s.emulateError(ModeSync, w.f.Type())
+	if err == nil {
+		w.f.s.stall(ModeSync, w.f.Type())
+		err = w.Writer.Sync()
+	}
+	w.f.s.count(ModeSync, w.f.Type(), 0)
+	if err != nil {
+		w.f.s.logI("sync error, num=%d type=%v err=%v", w.f.Num(), w.f.Type(), err)
+	}
+	return
+}
+
+func (w *writer) Close() (err error) {
+	return w.f.doClose(w.Writer)
+}
+
+type file struct {
+	s *Storage
+	storage.File
+}
+
+func (f *file) pack() uint64 {
+	return packFile(f.Num(), f.Type())
+}
+
+func (f *file) assertOpen() {
+	ExpectWithOffset(2, f.s.opens).NotTo(HaveKey(f.pack()), "File open, num=%d type=%v writer=%v", f.Num(), f.Type(), f.s.opens[f.pack()])
+}
+
+func (f *file) doClose(closer io.Closer) (err error) {
+	err = f.s.emulateError(ModeClose, f.Type())
+	if err == nil {
+		f.s.stall(ModeClose, f.Type())
+	}
+	f.s.mu.Lock()
+	defer f.s.mu.Unlock()
+	if err == nil {
+		ExpectWithOffset(2, f.s.opens).To(HaveKey(f.pack()), "File closed, num=%d type=%v", f.Num(), f.Type())
+		err = closer.Close()
+	}
+	f.s.countNB(ModeClose, f.Type(), 0)
+	writer := f.s.opens[f.pack()]
+	if err != nil {
+		f.s.logISkip(1, "file close failed, num=%d type=%v writer=%v err=%v", f.Num(), f.Type(), writer, err)
+	} else {
+		f.s.logISkip(1, "file closed, num=%d type=%v writer=%v", f.Num(), f.Type(), writer)
+		delete(f.s.opens, f.pack())
+	}
+	return
+}
+
+func (f *file) Open() (r storage.Reader, err error) {
+	err = f.s.emulateError(ModeOpen, f.Type())
+	if err == nil {
+		f.s.stall(ModeOpen, f.Type())
+	}
+	f.s.mu.Lock()
+	defer f.s.mu.Unlock()
+	if err == nil {
+		f.assertOpen()
+		f.s.countNB(ModeOpen, f.Type(), 0)
+		r, err = f.File.Open()
+	}
+	if err != nil {
+		f.s.logI("file open failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
+	} else {
+		f.s.logI("file opened, num=%d type=%v", f.Num(), f.Type())
+		f.s.opens[f.pack()] = false
+		r = &reader{f, r}
+	}
+	return
+}
+
+func (f *file) Create() (w storage.Writer, err error) {
+	err = f.s.emulateError(ModeCreate, f.Type())
+	if err == nil {
+		f.s.stall(ModeCreate, f.Type())
+	}
+	f.s.mu.Lock()
+	defer f.s.mu.Unlock()
+	if err == nil {
+		f.assertOpen()
+		f.s.countNB(ModeCreate, f.Type(), 0)
+		w, err = f.File.Create()
+	}
+	if err != nil {
+		f.s.logI("file create failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
+	} else {
+		f.s.logI("file created, num=%d type=%v", f.Num(), f.Type())
+		f.s.opens[f.pack()] = true
+		w = &writer{f, w}
+	}
+	return
+}
+
+func (f *file) Remove() (err error) {
+	err = f.s.emulateError(ModeRemove, f.Type())
+	if err == nil {
+		f.s.stall(ModeRemove, f.Type())
+	}
+	f.s.mu.Lock()
+	defer f.s.mu.Unlock()
+	if err == nil {
+		f.assertOpen()
+		f.s.countNB(ModeRemove, f.Type(), 0)
+		err = f.File.Remove()
+	}
+	if err != nil {
+		f.s.logI("file remove failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
+	} else {
+		f.s.logI("file removed, num=%d type=%v", f.Num(), f.Type())
+	}
+	return
+}
+
+type Storage struct {
+	storage.Storage
+	closeFn func() error
+
+	lmu sync.Mutex
+	lb  bytes.Buffer
+
+	mu sync.Mutex
+	// Open files, true=writer, false=reader
+	opens         map[uint64]bool
+	counters      [flattenCount]int
+	bytesCounter  [flattenCount]int64
+	emulatedError [flattenCount]error
+	stallCond     sync.Cond
+	stalled       [flattenCount]bool
+}
+
+func (s *Storage) log(skip int, str string) {
+	s.lmu.Lock()
+	defer s.lmu.Unlock()
+	_, file, line, ok := runtime.Caller(skip + 2)
+	if ok {
+		// Truncate file name at last file name separator.
+		if index := strings.LastIndex(file, "/"); index >= 0 {
+			file = file[index+1:]
+		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
+			file = file[index+1:]
+		}
+	} else {
+		file = "???"
+		line = 1
+	}
+	fmt.Fprintf(&s.lb, "%s:%d: ", file, line)
+	lines := strings.Split(str, "\n")
+	if l := len(lines); l > 1 && lines[l-1] == "" {
+		lines = lines[:l-1]
+	}
+	for i, line := range lines {
+		if i > 0 {
+			s.lb.WriteString("\n\t")
+		}
+		s.lb.WriteString(line)
+	}
+	s.lb.WriteByte('\n')
+}
+
+func (s *Storage) logISkip(skip int, format string, args ...interface{}) {
+	pc, _, _, ok := runtime.Caller(skip + 1)
+	if ok {
+		if f := runtime.FuncForPC(pc); f != nil {
+			fname := f.Name()
+			if index := strings.LastIndex(fname, "."); index >= 0 {
+				fname = fname[index+1:]
+			}
+			format = fname + ": " + format
+		}
+	}
+	s.log(skip+1, fmt.Sprintf(format, args...))
+}
+
+func (s *Storage) logI(format string, args ...interface{}) {
+	s.logISkip(1, format, args...)
+}
+
+func (s *Storage) Log(str string) {
+	s.log(1, "Log: "+str)
+}
+
+func (s *Storage) Lock() (r util.Releaser, err error) {
+	r, err = s.Storage.Lock()
+	if err != nil {
+		s.logI("storage locking failed, err=%v", err)
+	} else {
+		s.logI("storage locked")
+		r = storageLock{s, r}
+	}
+	return
+}
+
+func (s *Storage) GetFile(num uint64, t storage.FileType) storage.File {
+	return &file{s, s.Storage.GetFile(num, t)}
+}
+
+func (s *Storage) GetFiles(t storage.FileType) (files []storage.File, err error) {
+	rfiles, err := s.Storage.GetFiles(t)
+	if err != nil {
+		s.logI("get files failed, err=%v", err)
+		return
+	}
+	files = make([]storage.File, len(rfiles))
+	for i, f := range rfiles {
+		files[i] = &file{s, f}
+	}
+	s.logI("get files, type=0x%x count=%d", int(t), len(files))
+	return
+}
+
+func (s *Storage) GetManifest() (f storage.File, err error) {
+	manifest, err := s.Storage.GetManifest()
+	if err != nil {
+		if !os.IsNotExist(err) {
+			s.logI("get manifest failed, err=%v", err)
+		}
+		return
+	}
+	s.logI("get manifest, num=%d", manifest.Num())
+	return &file{s, manifest}, nil
+}
+
+func (s *Storage) SetManifest(f storage.File) error {
+	f_, ok := f.(*file)
+	ExpectWithOffset(1, ok).To(BeTrue())
+	ExpectWithOffset(1, f_.Type()).To(Equal(storage.TypeManifest))
+	err := s.Storage.SetManifest(f_.File)
+	if err != nil {
+		s.logI("set manifest failed, err=%v", err)
+	} else {
+		s.logI("set manifest, num=%d", f_.Num())
+	}
+	return err
+}
+
+func (s *Storage) openFiles() string {
+	out := "Open files:"
+	for x, writer := range s.opens {
+		num, t := unpackFile(x)
+		out += fmt.Sprintf("\n · num=%d type=%v writer=%v", num, t, writer)
+	}
+	return out
+}
+
+func (s *Storage) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles())
+	err := s.Storage.Close()
+	if err != nil {
+		s.logI("storage closing failed, err=%v", err)
+	} else {
+		s.logI("storage closed")
+	}
+	if s.closeFn != nil {
+		if err1 := s.closeFn(); err1 != nil {
+			s.logI("close func error, err=%v", err1)
+		}
+	}
+	return err
+}
+
+func (s *Storage) countNB(m StorageMode, t storage.FileType, n int) {
+	s.counters[flattenType(m, t)]++
+	s.bytesCounter[flattenType(m, t)] += int64(n)
+}
+
+func (s *Storage) count(m StorageMode, t storage.FileType, n int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.countNB(m, t, n)
+}
+
+func (s *Storage) ResetCounter(m StorageMode, t storage.FileType) {
+	for _, x := range listFlattenType(m, t) {
+		s.counters[x] = 0
+		s.bytesCounter[x] = 0
+	}
+}
+
+func (s *Storage) Counter(m StorageMode, t storage.FileType) (count int, bytes int64) {
+	for _, x := range listFlattenType(m, t) {
+		count += s.counters[x]
+		bytes += s.bytesCounter[x]
+	}
+	return
+}
+
+func (s *Storage) emulateError(m StorageMode, t storage.FileType) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	err := s.emulatedError[flattenType(m, t)]
+	if err != nil {
+		return emulatedError{err}
+	}
+	return nil
+}
+
+func (s *Storage) EmulateError(m StorageMode, t storage.FileType, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, x := range listFlattenType(m, t) {
+		s.emulatedError[x] = err
+	}
+}
+
+func (s *Storage) stall(m StorageMode, t storage.FileType) {
+	x := flattenType(m, t)
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for s.stalled[x] {
+		s.stallCond.Wait()
+	}
+}
+
+func (s *Storage) Stall(m StorageMode, t storage.FileType) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, x := range listFlattenType(m, t) {
+		s.stalled[x] = true
+	}
+}
+
+func (s *Storage) Release(m StorageMode, t storage.FileType) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, x := range listFlattenType(m, t) {
+		s.stalled[x] = false
+	}
+	s.stallCond.Broadcast()
+}
+
+func NewStorage() *Storage {
+	var stor storage.Storage
+	var closeFn func() error
+	if storageUseFS {
+		for {
+			storageMu.Lock()
+			num := storageNum
+			storageNum++
+			storageMu.Unlock()
+			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
+			if _, err := os.Stat(path); os.IsNotExist(err) {
+				stor, err = storage.OpenFile(path)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
+				closeFn = func() error {
+					if storageKeepFS {
+						return nil
+					}
+					return os.RemoveAll(path)
+				}
+				break
+			}
+		}
+	} else {
+		stor = storage.NewMemStorage()
+	}
+	s := &Storage{
+		Storage: stor,
+		closeFn: closeFn,
+		opens:   make(map[uint64]bool),
+	}
+	s.stallCond.L = &s.mu
+	return s
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package testutil
+
+import (
+	"bytes"
+	"flag"
+	"math/rand"
+	"reflect"
+	"sync"
+
+	"github.com/onsi/ginkgo/config"
+
+	"github.com/syndtr/goleveldb/leveldb/comparer"
+)
+
+var (
+	runfn = make(map[string][]func())
+	runmu sync.Mutex
+)
+
+func Defer(args ...interface{}) bool {
+	var (
+		group string
+		fn    func()
+	)
+	for _, arg := range args {
+		v := reflect.ValueOf(arg)
+		switch v.Kind() {
+		case reflect.String:
+			group = v.String()
+		case reflect.Func:
+			r := reflect.ValueOf(&fn).Elem()
+			r.Set(v)
+		}
+	}
+	if fn != nil {
+		runmu.Lock()
+		runfn[group] = append(runfn[group], fn)
+		runmu.Unlock()
+	}
+	return true
+}
+
+func RunDefer(groups ...string) bool {
+	if len(groups) == 0 {
+		groups = append(groups, "")
+	}
+	runmu.Lock()
+	var runfn_ []func()
+	for _, group := range groups {
+		runfn_ = append(runfn_, runfn[group]...)
+		delete(runfn, group)
+	}
+	runmu.Unlock()
+	for _, fn := range runfn_ {
+		fn()
+	}
+	return runfn_ != nil
+}
+
+func RandomSeed() int64 {
+	if !flag.Parsed() {
+		panic("random seed not initialized")
+	}
+	return config.GinkgoConfig.RandomSeed
+}
+
+func NewRand() *rand.Rand {
+	return rand.New(rand.NewSource(RandomSeed()))
+}
+
+var cmp = comparer.DefaultComparer
+
+func BytesSeparator(a, b []byte) []byte {
+	if bytes.Equal(a, b) {
+		return b
+	}
+	i, n := 0, len(a)
+	if n > len(b) {
+		n = len(b)
+	}
+	for ; i < n && (a[i] == b[i]); i++ {
+	}
+	x := append([]byte{}, a[:i]...)
+	if i < n {
+		if c := a[i] + 1; c < b[i] {
+			return append(x, c)
+		}
+		x = append(x, a[i])
+		i++
+	}
+	for ; i < len(a); i++ {
+		if c := a[i]; c < 0xff {
+			return append(x, c+1)
+		} else {
+			x = append(x, c)
+		}
+	}
+	if len(b) > i && b[i] > 0 {
+		return append(x, b[i]-1)
+	}
+	return append(x, 'x')
+}
+
+func BytesAfter(b []byte) []byte {
+	var x []byte
+	for _, c := range b {
+		if c < 0xff {
+			return append(x, c+1)
+		} else {
+			x = append(x, c)
+		}
+	}
+	return append(x, 'x')
+}
+
+func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
+	if rnd == nil {
+		rnd = NewRand()
+	}
+	for x := 0; x < round; x++ {
+		fn(rnd.Intn(n))
+	}
+	return
+}
+
+func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
+	if rnd == nil {
+		rnd = NewRand()
+	}
+	for x := 0; x < round; x++ {
+		for _, i := range rnd.Perm(n) {
+			fn(i)
+		}
+	}
+	return
+}
+
+func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) {
+	if rnd == nil {
+		rnd = NewRand()
+	}
+	for x := 0; x < round; x++ {
+		start := rnd.Intn(n)
+		length := 0
+		if j := n - start; j > 0 {
+			length = rnd.Intn(j)
+		}
+		fn(start, start+length)
+	}
+	return
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/testutil"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type testingDB struct {
+	*DB
+	ro   *opt.ReadOptions
+	wo   *opt.WriteOptions
+	stor *testutil.Storage
+}
+
+func (t *testingDB) TestPut(key []byte, value []byte) error {
+	return t.Put(key, value, t.wo)
+}
+
+func (t *testingDB) TestDelete(key []byte) error {
+	return t.Delete(key, t.wo)
+}
+
+func (t *testingDB) TestGet(key []byte) (value []byte, err error) {
+	return t.Get(key, t.ro)
+}
+
+func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator {
+	return t.NewIterator(slice, t.ro)
+}
+
+func (t *testingDB) TestClose() {
+	err := t.Close()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+	err = t.stor.Close()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+}
+
+func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB {
+	stor := testutil.NewStorage()
+	db, err := Open(stor, o)
+	Expect(err).NotTo(HaveOccurred())
+	return &testingDB{
+		DB:   db,
+		ro:   ro,
+		wo:   wo,
+		stor: stor,
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func shorten(str string) string {
+	if len(str) <= 4 {
+		return str
+	}
+	return str[:1] + ".." + str[len(str)-1:]
+}
+
+var bunits = [...]string{"", "Ki", "Mi", "Gi"}
+
+func shortenb(bytes int) string {
+	i := 0
+	for ; bytes > 1024 && i < 4; i++ {
+		bytes /= 1024
+	}
+	return fmt.Sprintf("%d%sB", bytes, bunits[i])
+}
+
+func sshortenb(bytes int) string {
+	if bytes == 0 {
+		return "~"
+	}
+	sign := "+"
+	if bytes < 0 {
+		sign = "-"
+		bytes *= -1
+	}
+	i := 0
+	for ; bytes > 1024 && i < 4; i++ {
+		bytes /= 1024
+	}
+	return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i])
+}
+
+func sint(x int) string {
+	if x == 0 {
+		return "~"
+	}
+	sign := "+"
+	if x < 0 {
+		sign = "-"
+		x *= -1
+	}
+	return fmt.Sprintf("%s%d", sign, x)
+}
+
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+type files []storage.File
+
+func (p files) Len() int {
+	return len(p)
+}
+
+func (p files) Less(i, j int) bool {
+	return p[i].Num() < p[j].Num()
+}
+
+func (p files) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (p files) sort() {
+	sort.Sort(p)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer.go

@@ -0,0 +1,293 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package util
+
+// This a copy of Go std bytes.Buffer with some modification
+// and some features stripped.
+
+import (
+	"bytes"
+	"io"
+)
+
+// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
+// The zero value for Buffer is an empty buffer ready to use.
+type Buffer struct {
+	buf       []byte   // contents are the bytes buf[off : len(buf)]
+	off       int      // read at &buf[off], write at &buf[len(buf)]
+	bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation.
+}
+
+// Bytes returns a slice of the contents of the unread portion of the buffer;
+// len(b.Bytes()) == b.Len().  If the caller changes the contents of the
+// returned slice, the contents of the buffer will change provided there
+// are no intervening method calls on the Buffer.
+func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
+
+// String returns the contents of the unread portion of the buffer
+// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
+func (b *Buffer) String() string {
+	if b == nil {
+		// Special case, useful in debugging.
+		return "<nil>"
+	}
+	return string(b.buf[b.off:])
+}
+
+// Len returns the number of bytes of the unread portion of the buffer;
+// b.Len() == len(b.Bytes()).
+func (b *Buffer) Len() int { return len(b.buf) - b.off }
+
+// Truncate discards all but the first n unread bytes from the buffer.
+// It panics if n is negative or greater than the length of the buffer.
+func (b *Buffer) Truncate(n int) {
+	switch {
+	case n < 0 || n > b.Len():
+		panic("leveldb/util.Buffer: truncation out of range")
+	case n == 0:
+		// Reuse buffer space.
+		b.off = 0
+	}
+	b.buf = b.buf[0 : b.off+n]
+}
+
+// Reset resets the buffer so it has no content.
+// b.Reset() is the same as b.Truncate(0).
+func (b *Buffer) Reset() { b.Truncate(0) }
+
+// grow grows the buffer to guarantee space for n more bytes.
+// It returns the index where bytes should be written.
+// If the buffer can't grow it will panic with bytes.ErrTooLarge.
+func (b *Buffer) grow(n int) int {
+	m := b.Len()
+	// If buffer is empty, reset to recover space.
+	if m == 0 && b.off != 0 {
+		b.Truncate(0)
+	}
+	if len(b.buf)+n > cap(b.buf) {
+		var buf []byte
+		if b.buf == nil && n <= len(b.bootstrap) {
+			buf = b.bootstrap[0:]
+		} else if m+n <= cap(b.buf)/2 {
+			// We can slide things down instead of allocating a new
+			// slice. We only need m+n <= cap(b.buf) to slide, but
+			// we instead let capacity get twice as large so we
+			// don't spend all our time copying.
+			copy(b.buf[:], b.buf[b.off:])
+			buf = b.buf[:m]
+		} else {
+			// not enough space anywhere
+			buf = makeSlice(2*cap(b.buf) + n)
+			copy(buf, b.buf[b.off:])
+		}
+		b.buf = buf
+		b.off = 0
+	}
+	b.buf = b.buf[0 : b.off+m+n]
+	return b.off + m
+}
+
+// Alloc allocs n bytes of slice from the buffer, growing the buffer as
+// needed. If n is negative, Alloc will panic.
+// If the buffer can't grow it will panic with bytes.ErrTooLarge.
+func (b *Buffer) Alloc(n int) []byte {
+	if n < 0 {
+		panic("leveldb/util.Buffer.Alloc: negative count")
+	}
+	m := b.grow(n)
+	return b.buf[m:]
+}
+
+// Grow grows the buffer's capacity, if necessary, to guarantee space for
+// another n bytes. After Grow(n), at least n bytes can be written to the
+// buffer without another allocation.
+// If n is negative, Grow will panic.
+// If the buffer can't grow it will panic with bytes.ErrTooLarge.
+func (b *Buffer) Grow(n int) {
+	if n < 0 {
+		panic("leveldb/util.Buffer.Grow: negative count")
+	}
+	m := b.grow(n)
+	b.buf = b.buf[0:m]
+}
+
+// Write appends the contents of p to the buffer, growing the buffer as
+// needed. The return value n is the length of p; err is always nil. If the
+// buffer becomes too large, Write will panic with bytes.ErrTooLarge.
+func (b *Buffer) Write(p []byte) (n int, err error) {
+	m := b.grow(len(p))
+	return copy(b.buf[m:], p), nil
+}
+
+// MinRead is the minimum slice size passed to a Read call by
+// Buffer.ReadFrom.  As long as the Buffer has at least MinRead bytes beyond
+// what is required to hold the contents of r, ReadFrom will not grow the
+// underlying buffer.
+const MinRead = 512
+
+// ReadFrom reads data from r until EOF and appends it to the buffer, growing
+// the buffer as needed. The return value n is the number of bytes read. Any
+// error except io.EOF encountered during the read is also returned. If the
+// buffer becomes too large, ReadFrom will panic with bytes.ErrTooLarge.
+func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
+	// If buffer is empty, reset to recover space.
+	if b.off >= len(b.buf) {
+		b.Truncate(0)
+	}
+	for {
+		if free := cap(b.buf) - len(b.buf); free < MinRead {
+			// not enough space at end
+			newBuf := b.buf
+			if b.off+free < MinRead {
+				// not enough space using beginning of buffer;
+				// double buffer capacity
+				newBuf = makeSlice(2*cap(b.buf) + MinRead)
+			}
+			copy(newBuf, b.buf[b.off:])
+			b.buf = newBuf[:len(b.buf)-b.off]
+			b.off = 0
+		}
+		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
+		b.buf = b.buf[0 : len(b.buf)+m]
+		n += int64(m)
+		if e == io.EOF {
+			break
+		}
+		if e != nil {
+			return n, e
+		}
+	}
+	return n, nil // err is EOF, so return nil explicitly
+}
+
+// makeSlice allocates a slice of size n. If the allocation fails, it panics
+// with bytes.ErrTooLarge.
+func makeSlice(n int) []byte {
+	// If the make fails, give a known error.
+	defer func() {
+		if recover() != nil {
+			panic(bytes.ErrTooLarge)
+		}
+	}()
+	return make([]byte, n)
+}
+
+// WriteTo writes data to w until the buffer is drained or an error occurs.
+// The return value n is the number of bytes written; it always fits into an
+// int, but it is int64 to match the io.WriterTo interface. Any error
+// encountered during the write is also returned.
+func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
+	if b.off < len(b.buf) {
+		nBytes := b.Len()
+		m, e := w.Write(b.buf[b.off:])
+		if m > nBytes {
+			panic("leveldb/util.Buffer.WriteTo: invalid Write count")
+		}
+		b.off += m
+		n = int64(m)
+		if e != nil {
+			return n, e
+		}
+		// all bytes should have been written, by definition of
+		// Write method in io.Writer
+		if m != nBytes {
+			return n, io.ErrShortWrite
+		}
+	}
+	// Buffer is now empty; reset.
+	b.Truncate(0)
+	return
+}
+
+// WriteByte appends the byte c to the buffer, growing the buffer as needed.
+// The returned error is always nil, but is included to match bufio.Writer's
+// WriteByte. If the buffer becomes too large, WriteByte will panic with
+// bytes.ErrTooLarge.
+func (b *Buffer) WriteByte(c byte) error {
+	m := b.grow(1)
+	b.buf[m] = c
+	return nil
+}
+
+// Read reads the next len(p) bytes from the buffer or until the buffer
+// is drained.  The return value n is the number of bytes read.  If the
+// buffer has no data to return, err is io.EOF (unless len(p) is zero);
+// otherwise it is nil.
+func (b *Buffer) Read(p []byte) (n int, err error) {
+	if b.off >= len(b.buf) {
+		// Buffer is empty, reset to recover space.
+		b.Truncate(0)
+		if len(p) == 0 {
+			return
+		}
+		return 0, io.EOF
+	}
+	n = copy(p, b.buf[b.off:])
+	b.off += n
+	return
+}
+
+// Next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
+// The slice is only valid until the next call to a read or write method.
+func (b *Buffer) Next(n int) []byte {
+	m := b.Len()
+	if n > m {
+		n = m
+	}
+	data := b.buf[b.off : b.off+n]
+	b.off += n
+	return data
+}
+
+// ReadByte reads and returns the next byte from the buffer.
+// If no byte is available, it returns error io.EOF.
+func (b *Buffer) ReadByte() (c byte, err error) {
+	if b.off >= len(b.buf) {
+		// Buffer is empty, reset to recover space.
+		b.Truncate(0)
+		return 0, io.EOF
+	}
+	c = b.buf[b.off]
+	b.off++
+	return c, nil
+}
+
+// ReadBytes reads until the first occurrence of delim in the input,
+// returning a slice containing the data up to and including the delimiter.
+// If ReadBytes encounters an error before finding a delimiter,
+// it returns the data read before the error and the error itself (often io.EOF).
+// ReadBytes returns err != nil if and only if the returned data does not end in
+// delim.
+func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
+	slice, err := b.readSlice(delim)
+	// return a copy of slice. The buffer's backing array may
+	// be overwritten by later calls.
+	line = append(line, slice...)
+	return
+}
+
+// readSlice is like ReadBytes but returns a reference to internal buffer data.
+func (b *Buffer) readSlice(delim byte) (line []byte, err error) {
+	i := bytes.IndexByte(b.buf[b.off:], delim)
+	end := b.off + i + 1
+	if i < 0 {
+		end = len(b.buf)
+		err = io.EOF
+	}
+	line = b.buf[b.off:end]
+	b.off = end
+	return line, err
+}
+
+// NewBuffer creates and initializes a new Buffer using buf as its initial
+// contents.  It is intended to prepare a Buffer to read existing data.  It
+// can also be used to size the internal buffer for writing. To do that,
+// buf should have the desired capacity but a length of zero.
+//
+// In most cases, new(Buffer) (or just declaring a Buffer variable) is
+// sufficient to initialize a Buffer.
+func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go

@@ -0,0 +1,369 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import (
+	"bytes"
+	"io"
+	"math/rand"
+	"runtime"
+	"testing"
+)
+
+const N = 10000      // make this bigger for a larger (and slower) test
+var data string      // test data for write tests
+var testBytes []byte // test data; same as data but as a slice.
+
+func init() {
+	testBytes = make([]byte, N)
+	for i := 0; i < N; i++ {
+		testBytes[i] = 'a' + byte(i%26)
+	}
+	data = string(testBytes)
+}
+
+// Verify that contents of buf match the string s.
+func check(t *testing.T, testname string, buf *Buffer, s string) {
+	bytes := buf.Bytes()
+	str := buf.String()
+	if buf.Len() != len(bytes) {
+		t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes))
+	}
+
+	if buf.Len() != len(str) {
+		t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str))
+	}
+
+	if buf.Len() != len(s) {
+		t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s))
+	}
+
+	if string(bytes) != s {
+		t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s)
+	}
+}
+
+// Fill buf through n writes of byte slice fub.
+// The initial contents of buf corresponds to the string s;
+// the result is the final contents of buf returned as a string.
+func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string {
+	check(t, testname+" (fill 1)", buf, s)
+	for ; n > 0; n-- {
+		m, err := buf.Write(fub)
+		if m != len(fub) {
+			t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub))
+		}
+		if err != nil {
+			t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err)
+		}
+		s += string(fub)
+		check(t, testname+" (fill 4)", buf, s)
+	}
+	return s
+}
+
+func TestNewBuffer(t *testing.T) {
+	buf := NewBuffer(testBytes)
+	check(t, "NewBuffer", buf, data)
+}
+
+// Empty buf through repeated reads into fub.
+// The initial contents of buf corresponds to the string s.
+func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) {
+	check(t, testname+" (empty 1)", buf, s)
+
+	for {
+		n, err := buf.Read(fub)
+		if n == 0 {
+			break
+		}
+		if err != nil {
+			t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err)
+		}
+		s = s[n:]
+		check(t, testname+" (empty 3)", buf, s)
+	}
+
+	check(t, testname+" (empty 4)", buf, "")
+}
+
+func TestBasicOperations(t *testing.T) {
+	var buf Buffer
+
+	for i := 0; i < 5; i++ {
+		check(t, "TestBasicOperations (1)", &buf, "")
+
+		buf.Reset()
+		check(t, "TestBasicOperations (2)", &buf, "")
+
+		buf.Truncate(0)
+		check(t, "TestBasicOperations (3)", &buf, "")
+
+		n, err := buf.Write([]byte(data[0:1]))
+		if n != 1 {
+			t.Errorf("wrote 1 byte, but n == %d", n)
+		}
+		if err != nil {
+			t.Errorf("err should always be nil, but err == %s", err)
+		}
+		check(t, "TestBasicOperations (4)", &buf, "a")
+
+		buf.WriteByte(data[1])
+		check(t, "TestBasicOperations (5)", &buf, "ab")
+
+		n, err = buf.Write([]byte(data[2:26]))
+		if n != 24 {
+			t.Errorf("wrote 25 bytes, but n == %d", n)
+		}
+		check(t, "TestBasicOperations (6)", &buf, string(data[0:26]))
+
+		buf.Truncate(26)
+		check(t, "TestBasicOperations (7)", &buf, string(data[0:26]))
+
+		buf.Truncate(20)
+		check(t, "TestBasicOperations (8)", &buf, string(data[0:20]))
+
+		empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5))
+		empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100))
+
+		buf.WriteByte(data[1])
+		c, err := buf.ReadByte()
+		if err != nil {
+			t.Error("ReadByte unexpected eof")
+		}
+		if c != data[1] {
+			t.Errorf("ReadByte wrong value c=%v", c)
+		}
+		c, err = buf.ReadByte()
+		if err == nil {
+			t.Error("ReadByte unexpected not eof")
+		}
+	}
+}
+
+func TestLargeByteWrites(t *testing.T) {
+	var buf Buffer
+	limit := 30
+	if testing.Short() {
+		limit = 9
+	}
+	for i := 3; i < limit; i += 3 {
+		s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes)
+		empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i))
+	}
+	check(t, "TestLargeByteWrites (3)", &buf, "")
+}
+
+func TestLargeByteReads(t *testing.T) {
+	var buf Buffer
+	for i := 3; i < 30; i += 3 {
+		s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
+		empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
+	}
+	check(t, "TestLargeByteReads (3)", &buf, "")
+}
+
+func TestMixedReadsAndWrites(t *testing.T) {
+	var buf Buffer
+	s := ""
+	for i := 0; i < 50; i++ {
+		wlen := rand.Intn(len(data))
+		s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen])
+		rlen := rand.Intn(len(data))
+		fub := make([]byte, rlen)
+		n, _ := buf.Read(fub)
+		s = s[n:]
+	}
+	empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len()))
+}
+
+func TestNil(t *testing.T) {
+	var b *Buffer
+	if b.String() != "<nil>" {
+		t.Errorf("expected <nil>; got %q", b.String())
+	}
+}
+
+func TestReadFrom(t *testing.T) {
+	var buf Buffer
+	for i := 3; i < 30; i += 3 {
+		s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
+		var b Buffer
+		b.ReadFrom(&buf)
+		empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
+	}
+}
+
+func TestWriteTo(t *testing.T) {
+	var buf Buffer
+	for i := 3; i < 30; i += 3 {
+		s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
+		var b Buffer
+		buf.WriteTo(&b)
+		empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
+	}
+}
+
+func TestNext(t *testing.T) {
+	b := []byte{0, 1, 2, 3, 4}
+	tmp := make([]byte, 5)
+	for i := 0; i <= 5; i++ {
+		for j := i; j <= 5; j++ {
+			for k := 0; k <= 6; k++ {
+				// 0 <= i <= j <= 5; 0 <= k <= 6
+				// Check that if we start with a buffer
+				// of length j at offset i and ask for
+				// Next(k), we get the right bytes.
+				buf := NewBuffer(b[0:j])
+				n, _ := buf.Read(tmp[0:i])
+				if n != i {
+					t.Fatalf("Read %d returned %d", i, n)
+				}
+				bb := buf.Next(k)
+				want := k
+				if want > j-i {
+					want = j - i
+				}
+				if len(bb) != want {
+					t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb))
+				}
+				for l, v := range bb {
+					if v != byte(l+i) {
+						t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i)
+					}
+				}
+			}
+		}
+	}
+}
+
+var readBytesTests = []struct {
+	buffer   string
+	delim    byte
+	expected []string
+	err      error
+}{
+	{"", 0, []string{""}, io.EOF},
+	{"a\x00", 0, []string{"a\x00"}, nil},
+	{"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil},
+	{"hello\x01world", 1, []string{"hello\x01"}, nil},
+	{"foo\nbar", 0, []string{"foo\nbar"}, io.EOF},
+	{"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil},
+	{"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, io.EOF},
+}
+
+func TestReadBytes(t *testing.T) {
+	for _, test := range readBytesTests {
+		buf := NewBuffer([]byte(test.buffer))
+		var err error
+		for _, expected := range test.expected {
+			var bytes []byte
+			bytes, err = buf.ReadBytes(test.delim)
+			if string(bytes) != expected {
+				t.Errorf("expected %q, got %q", expected, bytes)
+			}
+			if err != nil {
+				break
+			}
+		}
+		if err != test.err {
+			t.Errorf("expected error %v, got %v", test.err, err)
+		}
+	}
+}
+
+func TestGrow(t *testing.T) {
+	x := []byte{'x'}
+	y := []byte{'y'}
+	tmp := make([]byte, 72)
+	for _, startLen := range []int{0, 100, 1000, 10000, 100000} {
+		xBytes := bytes.Repeat(x, startLen)
+		for _, growLen := range []int{0, 100, 1000, 10000, 100000} {
+			buf := NewBuffer(xBytes)
+			// If we read, this affects buf.off, which is good to test.
+			readBytes, _ := buf.Read(tmp)
+			buf.Grow(growLen)
+			yBytes := bytes.Repeat(y, growLen)
+			// Check no allocation occurs in write, as long as we're single-threaded.
+			var m1, m2 runtime.MemStats
+			runtime.ReadMemStats(&m1)
+			buf.Write(yBytes)
+			runtime.ReadMemStats(&m2)
+			if runtime.GOMAXPROCS(-1) == 1 && m1.Mallocs != m2.Mallocs {
+				t.Errorf("allocation occurred during write")
+			}
+			// Check that buffer has correct data.
+			if !bytes.Equal(buf.Bytes()[0:startLen-readBytes], xBytes[readBytes:]) {
+				t.Errorf("bad initial data at %d %d", startLen, growLen)
+			}
+			if !bytes.Equal(buf.Bytes()[startLen-readBytes:startLen-readBytes+growLen], yBytes) {
+				t.Errorf("bad written data at %d %d", startLen, growLen)
+			}
+		}
+	}
+}
+
+// Was a bug: used to give EOF reading empty slice at EOF.
+func TestReadEmptyAtEOF(t *testing.T) {
+	b := new(Buffer)
+	slice := make([]byte, 0)
+	n, err := b.Read(slice)
+	if err != nil {
+		t.Errorf("read error: %v", err)
+	}
+	if n != 0 {
+		t.Errorf("wrong count; got %d want 0", n)
+	}
+}
+
+// Tests that we occasionally compact. Issue 5154.
+func TestBufferGrowth(t *testing.T) {
+	var b Buffer
+	buf := make([]byte, 1024)
+	b.Write(buf[0:1])
+	var cap0 int
+	for i := 0; i < 5<<10; i++ {
+		b.Write(buf)
+		b.Read(buf)
+		if i == 0 {
+			cap0 = cap(b.buf)
+		}
+	}
+	cap1 := cap(b.buf)
+	// (*Buffer).grow allows for 2x capacity slop before sliding,
+	// so set our error threshold at 3x.
+	if cap1 > cap0*3 {
+		t.Errorf("buffer cap = %d; too big (grew from %d)", cap1, cap0)
+	}
+}
+
+// From Issue 5154.
+func BenchmarkBufferNotEmptyWriteRead(b *testing.B) {
+	buf := make([]byte, 1024)
+	for i := 0; i < b.N; i++ {
+		var b Buffer
+		b.Write(buf[0:1])
+		for i := 0; i < 5<<10; i++ {
+			b.Write(buf)
+			b.Read(buf)
+		}
+	}
+}
+
+// Check that we don't compact too often. From Issue 5154.
+func BenchmarkBufferFullSmallReads(b *testing.B) {
+	buf := make([]byte, 1024)
+	for i := 0; i < b.N; i++ {
+		var b Buffer
+		b.Write(buf)
+		for b.Len()+20 < cap(b.buf) {
+			b.Write(buf[:10])
+		}
+		for i := 0; i < 5<<10; i++ {
+			b.Read(buf[:1])
+			b.Write(buf[:1])
+		}
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/crc32.go

@@ -0,0 +1,30 @@
+// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package util
+
+import (
+	"hash/crc32"
+)
+
+var table = crc32.MakeTable(crc32.Castagnoli)
+
+// CRC is a CRC-32 checksum computed using Castagnoli's polynomial.
+type CRC uint32
+
+// NewCRC creates a new crc based on the given bytes.
+func NewCRC(b []byte) CRC {
+	return CRC(0).Update(b)
+}
+
+// Update updates the crc with the given bytes.
+func (c CRC) Update(b []byte) CRC {
+	return CRC(crc32.Update(uint32(c), table, b))
+}
+
+// Value returns a masked crc.
+func (c CRC) Value() uint32 {
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/hash.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package util
+
+import (
+	"bytes"
+	"encoding/binary"
+)
+
+// Hash return hash of the given data.
+func Hash(data []byte, seed uint32) uint32 {
+	// Similar to murmur hash
+	var m uint32 = 0xc6a4a793
+	var r uint32 = 24
+	h := seed ^ (uint32(len(data)) * m)
+
+	buf := bytes.NewBuffer(data)
+	for buf.Len() >= 4 {
+		var w uint32
+		binary.Read(buf, binary.LittleEndian, &w)
+		h += w
+		h *= m
+		h ^= (h >> 16)
+	}
+
+	rest := buf.Bytes()
+	switch len(rest) {
+	default:
+		panic("not reached")
+	case 3:
+		h += uint32(rest[2]) << 16
+		fallthrough
+	case 2:
+		h += uint32(rest[1]) << 8
+		fallthrough
+	case 1:
+		h += uint32(rest[0])
+		h *= m
+		h ^= (h >> r)
+	case 0:
+	}
+
+	return h
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package util
+
+// Range is a key range.
+type Range struct {
+	// Start of the key range, include in the range.
+	Start []byte
+
+	// Limit of the key range, not include in the range.
+	Limit []byte
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package util provides utilities used throughout leveldb.
+package util
+
+import (
+	"errors"
+)
+
+var (
+	ErrNotFound = errors.New("leveldb: not found")
+)
+
+// Releaser is the interface that wraps the basic Release method.
+type Releaser interface {
+	// Release releases associated resources. Release should always success
+	// and can be called multipe times without causing error.
+	Release()
+}
+
+// ReleaseSetter is the interface that wraps the basic SetReleaser method.
+type ReleaseSetter interface {
+	// SetReleaser associates the given releaser to the resources. The
+	// releaser will be called once coresponding resources released.
+	// Calling SetReleaser with nil will clear the releaser.
+	SetReleaser(releaser Releaser)
+}
+
+// BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
+type BasicReleaser struct {
+	releaser Releaser
+}
+
+// Release implements Releaser.Release.
+func (r *BasicReleaser) Release() {
+	if r.releaser != nil {
+		r.releaser.Release()
+		r.releaser = nil
+	}
+}
+
+// SetReleaser implements ReleaseSetter.SetReleaser.
+func (r *BasicReleaser) SetReleaser(releaser Releaser) {
+	r.releaser = releaser
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go

@@ -0,0 +1,437 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"errors"
+	"sync/atomic"
+	"unsafe"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var levelMaxSize [kNumLevels]float64
+
+func init() {
+	// Precompute max size of each level
+	for level := range levelMaxSize {
+		res := float64(10 * 1048576)
+		for n := level; n > 1; n-- {
+			res *= 10
+		}
+		levelMaxSize[level] = res
+	}
+}
+
+type tSet struct {
+	level int
+	table *tFile
+}
+
+type version struct {
+	s *session
+
+	tables [kNumLevels]tFiles
+
+	// Level that should be compacted next and its compaction score.
+	// Score < 1 means compaction is not strictly needed.  These fields
+	// are initialized by ComputeCompaction()
+	cLevel int
+	cScore float64
+
+	cSeek unsafe.Pointer
+
+	ref  int
+	next *version
+}
+
+func (v *version) release_NB() {
+	v.ref--
+	if v.ref > 0 {
+		return
+	}
+	if v.ref < 0 {
+		panic("negative version ref")
+	}
+
+	s := v.s
+
+	tables := make(map[uint64]bool)
+	for _, tt := range v.next.tables {
+		for _, t := range tt {
+			num := t.file.Num()
+			tables[num] = true
+		}
+	}
+
+	for _, tt := range v.tables {
+		for _, t := range tt {
+			num := t.file.Num()
+			if _, ok := tables[num]; !ok {
+				s.tops.remove(t)
+			}
+		}
+	}
+
+	v.next.release_NB()
+	v.next = nil
+}
+
+func (v *version) release() {
+	v.s.vmu.Lock()
+	v.release_NB()
+	v.s.vmu.Unlock()
+}
+
+func (v *version) get(key iKey, ro *opt.ReadOptions) (value []byte, cstate bool, err error) {
+	s := v.s
+	icmp := s.cmp
+	ucmp := icmp.cmp
+
+	ukey := key.ukey()
+
+	var tset *tSet
+	tseek := true
+
+	// We can search level-by-level since entries never hop across
+	// levels. Therefore we are guaranteed that if we find data
+	// in an smaller level, later levels are irrelevant.
+	for level, ts := range v.tables {
+		if len(ts) == 0 {
+			continue
+		}
+
+		if level == 0 {
+			// Level-0 files may overlap each other. Find all files that
+			// overlap user_key and process them in order from newest to
+			var tmp tFiles
+			for _, t := range ts {
+				if ucmp.Compare(ukey, t.min.ukey()) >= 0 &&
+					ucmp.Compare(ukey, t.max.ukey()) <= 0 {
+					tmp = append(tmp, t)
+				}
+			}
+
+			if len(tmp) == 0 {
+				continue
+			}
+
+			tmp.sortByNum()
+			ts = tmp
+		} else {
+			i := ts.searchMax(key, icmp)
+			if i >= len(ts) || ucmp.Compare(ukey, ts[i].min.ukey()) < 0 {
+				continue
+			}
+
+			ts = ts[i : i+1]
+		}
+
+		var l0found bool
+		var l0seq uint64
+		var l0type vType
+		var l0value []byte
+		for _, t := range ts {
+			if tseek {
+				if tset == nil {
+					tset = &tSet{level, t}
+				} else if tset.table.incrSeek() <= 0 {
+					cstate = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
+					tseek = false
+				}
+			}
+
+			var _rkey, rval []byte
+			_rkey, rval, err = s.tops.get(t, key, ro)
+			if err == ErrNotFound {
+				continue
+			} else if err != nil {
+				return
+			}
+
+			rkey := iKey(_rkey)
+			if seq, t, ok := rkey.parseNum(); ok {
+				if ucmp.Compare(ukey, rkey.ukey()) == 0 {
+					if level == 0 {
+						if seq >= l0seq {
+							l0found = true
+							l0seq = seq
+							l0type = t
+							l0value = rval
+						}
+					} else {
+						switch t {
+						case tVal:
+							value = rval
+						case tDel:
+							err = ErrNotFound
+						default:
+							panic("invalid type")
+						}
+						return
+					}
+				}
+			} else {
+				err = errors.New("leveldb: internal key corrupted")
+				return
+			}
+		}
+		if level == 0 && l0found {
+			switch l0type {
+			case tVal:
+				value = l0value
+			case tDel:
+				err = ErrNotFound
+			default:
+				panic("invalid type")
+			}
+			return
+		}
+	}
+
+	err = ErrNotFound
+	return
+}
+
+func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) {
+	s := v.s
+	icmp := s.cmp
+
+	// Merge all level zero files together since they may overlap
+	for _, t := range v.tables[0] {
+		it := s.tops.newIterator(t, slice, ro)
+		its = append(its, it)
+	}
+
+	strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
+	for _, tt := range v.tables[1:] {
+		if len(tt) == 0 {
+			continue
+		}
+
+		it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, icmp, slice, ro), strict, true)
+		its = append(its, it)
+	}
+
+	return
+}
+
+func (v *version) newStaging() *versionStaging {
+	return &versionStaging{base: v}
+}
+
+// Spawn a new version based on this version.
+func (v *version) spawn(r *sessionRecord) *version {
+	staging := v.newStaging()
+	staging.commit(r)
+	return staging.finish()
+}
+
+func (v *version) fillRecord(r *sessionRecord) {
+	for level, ts := range v.tables {
+		for _, t := range ts {
+			r.addTableFile(level, t)
+		}
+	}
+}
+
+func (v *version) tLen(level int) int {
+	return len(v.tables[level])
+}
+
+func (v *version) getApproximateOffset(key iKey) (n uint64, err error) {
+	icmp := v.s.cmp
+	tops := v.s.tops
+
+	for level, tt := range v.tables {
+		for _, t := range tt {
+			if icmp.Compare(t.max, key) <= 0 {
+				// Entire file is before "key", so just add the file size
+				n += t.size
+			} else if icmp.Compare(t.min, key) > 0 {
+				// Entire file is after "key", so ignore
+				if level > 0 {
+					// Files other than level 0 are sorted by meta->min, so
+					// no further files in this level will contain data for
+					// "key".
+					break
+				}
+			} else {
+				// "key" falls in the range for this table.  Add the
+				// approximate offset of "key" within the table.
+				var nn uint64
+				nn, err = tops.getApproximateOffset(t, key)
+				if err != nil {
+					return 0, err
+				}
+				n += nn
+			}
+		}
+	}
+
+	return
+}
+
+func (v *version) pickLevel(min, max []byte) (level int) {
+	icmp := v.s.cmp
+	ucmp := icmp.cmp
+
+	if !v.tables[0].isOverlaps(min, max, false, icmp) {
+		var r tFiles
+		for ; level < kMaxMemCompactLevel; level++ {
+			if v.tables[level+1].isOverlaps(min, max, true, icmp) {
+				break
+			}
+			v.tables[level+2].getOverlaps(min, max, &r, true, ucmp)
+			if r.size() > kMaxGrandParentOverlapBytes {
+				break
+			}
+		}
+	}
+
+	return
+}
+
+func (v *version) computeCompaction() {
+	// Precomputed best level for next compaction
+	var bestLevel int = -1
+	var bestScore float64 = -1
+
+	for level, ff := range v.tables {
+		var score float64
+		if level == 0 {
+			// We treat level-0 specially by bounding the number of files
+			// instead of number of bytes for two reasons:
+			//
+			// (1) With larger write-buffer sizes, it is nice not to do too
+			// many level-0 compactions.
+			//
+			// (2) The files in level-0 are merged on every read and
+			// therefore we wish to avoid too many files when the individual
+			// file size is small (perhaps because of a small write-buffer
+			// setting, or very high compression ratios, or lots of
+			// overwrites/deletions).
+			score = float64(len(ff)) / kL0_CompactionTrigger
+		} else {
+			score = float64(ff.size()) / levelMaxSize[level]
+		}
+
+		if score > bestScore {
+			bestLevel = level
+			bestScore = score
+		}
+	}
+
+	v.cLevel = bestLevel
+	v.cScore = bestScore
+}
+
+func (v *version) needCompaction() bool {
+	return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
+}
+
+type versionStaging struct {
+	base   *version
+	tables [kNumLevels]struct {
+		added   map[uint64]ntRecord
+		deleted map[uint64]struct{}
+	}
+}
+
+func (p *versionStaging) commit(r *sessionRecord) {
+	btt := p.base.tables
+
+	// deleted tables
+	for _, tr := range r.deletedTables {
+		tm := &(p.tables[tr.level])
+
+		bt := btt[tr.level]
+		if len(bt) > 0 {
+			if tm.deleted == nil {
+				tm.deleted = make(map[uint64]struct{})
+			}
+			tm.deleted[tr.num] = struct{}{}
+		}
+
+		if tm.added != nil {
+			delete(tm.added, tr.num)
+		}
+	}
+
+	// new tables
+	for _, tr := range r.addedTables {
+		tm := &(p.tables[tr.level])
+
+		if tm.added == nil {
+			tm.added = make(map[uint64]ntRecord)
+		}
+		tm.added[tr.num] = tr
+
+		if tm.deleted != nil {
+			delete(tm.deleted, tr.num)
+		}
+	}
+}
+
+func (p *versionStaging) finish() *version {
+	s := p.base.s
+	btt := p.base.tables
+
+	// build new version
+	nv := &version{s: s}
+	for level, tm := range p.tables {
+		bt := btt[level]
+
+		n := len(bt) + len(tm.added) - len(tm.deleted)
+		if n < 0 {
+			n = 0
+		}
+		nt := make(tFiles, 0, n)
+
+		// base tables
+		for _, t := range bt {
+			if _, ok := tm.deleted[t.file.Num()]; ok {
+				continue
+			}
+			if _, ok := tm.added[t.file.Num()]; ok {
+				continue
+			}
+			nt = append(nt, t)
+		}
+
+		// new tables
+		for _, tr := range tm.added {
+			nt = append(nt, tr.makeFile(s))
+		}
+
+		// sort tables
+		nt.sortByKey(s.cmp)
+		nv.tables[level] = nt
+	}
+
+	// compute compaction score for new version
+	nv.computeCompaction()
+
+	return nv
+}
+
+type versionReleaser struct {
+	v    *version
+	once bool
+}
+
+func (vr *versionReleaser) Release() {
+	v := vr.v
+	v.s.vmu.Lock()
+	if !vr.once {
+		v.release_NB()
+		vr.once = true
+	}
+	v.s.vmu.Unlock()
+}

cid/cid.go

@@ -1,87 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-// Use of this source code is governed by an MIT-style license that can be
-// found in the LICENSE file.
-
-// Package cid provides a manager for mappings between node ID:s and connection ID:s.
-package cid
-
-import (
-	"sync"
-
-	"github.com/calmh/syncthing/protocol"
-)
-
-type Map struct {
-	sync.Mutex
-	toCid  map[protocol.NodeID]uint
-	toName []protocol.NodeID
-}
-
-var (
-	LocalNodeID      = protocol.NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
-	LocalID     uint = 0
-	emptyNodeID protocol.NodeID
-)
-
-func NewMap() *Map {
-	return &Map{
-		toCid:  map[protocol.NodeID]uint{LocalNodeID: LocalID},
-		toName: []protocol.NodeID{LocalNodeID},
-	}
-}
-
-func (m *Map) Get(name protocol.NodeID) uint {
-	m.Lock()
-	defer m.Unlock()
-
-	cid, ok := m.toCid[name]
-	if ok {
-		return cid
-	}
-
-	// Find a free slot to get a new ID
-	for i, n := range m.toName {
-		if n == emptyNodeID {
-			m.toName[i] = name
-			m.toCid[name] = uint(i)
-			return uint(i)
-		}
-	}
-
-	// Add it to the end since we didn't find a free slot
-	m.toName = append(m.toName, name)
-	cid = uint(len(m.toName) - 1)
-	m.toCid[name] = cid
-	return cid
-}
-
-func (m *Map) Name(cid uint) protocol.NodeID {
-	m.Lock()
-	defer m.Unlock()
-
-	return m.toName[cid]
-}
-
-func (m *Map) Names() []protocol.NodeID {
-	m.Lock()
-
-	var names []protocol.NodeID
-	for _, name := range m.toName {
-		if name != emptyNodeID {
-			names = append(names, name)
-		}
-	}
-
-	m.Unlock()
-	return names
-}
-
-func (m *Map) Clear(name protocol.NodeID) {
-	m.Lock()
-	cid, ok := m.toCid[name]
-	if ok {
-		m.toName[cid] = emptyNodeID
-		delete(m.toCid, name)
-	}
-	m.Unlock()
-}

cid/cid_test.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-// Use of this source code is governed by an MIT-style license that can be
-// found in the LICENSE file.
-
-package cid
-
-import (
-	"testing"
-
-	"github.com/calmh/syncthing/protocol"
-)
-
-func TestGet(t *testing.T) {
-	m := NewMap()
-
-	fooID := protocol.NewNodeID([]byte("foo"))
-	barID := protocol.NewNodeID([]byte("bar"))
-
-	if i := m.Get(fooID); i != 1 {
-		t.Errorf("Unexpected id %d != 1", i)
-	}
-	if i := m.Get(barID); i != 2 {
-		t.Errorf("Unexpected id %d != 2", i)
-	}
-	if i := m.Get(fooID); i != 1 {
-		t.Errorf("Unexpected id %d != 1", i)
-	}
-	if i := m.Get(barID); i != 2 {
-		t.Errorf("Unexpected id %d != 2", i)
-	}
-
-	if LocalID != 0 {
-		t.Error("LocalID should be 0")
-	}
-	if i := m.Get(LocalNodeID); i != LocalID {
-		t.Errorf("Unexpected id %d != %d", i, LocalID)
-	}
-}

cmd/syncthing/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/upnp"
 	"github.com/juju/ratelimit"
+	"github.com/syndtr/goleveldb/leveldb"
 )
 
 var (
@@ -288,7 +289,16 @@ func main() {
 		rateBucket = ratelimit.NewBucketWithRate(float64(1000*cfg.Options.MaxSendKbps), int64(5*1000*cfg.Options.MaxSendKbps))
 	}
 
-	m := model.NewModel(confDir, &cfg, "syncthing", Version)
+	havePersistentIndex := false
+	if fi, err := os.Stat(filepath.Join(confDir, "index")); err == nil && fi.IsDir() {
+		havePersistentIndex = true
+	}
+
+	db, err := leveldb.OpenFile(filepath.Join(confDir, "index"), nil)
+	if err != nil {
+		l.Fatalln("leveldb.OpenFile():", err)
+	}
+	m := model.NewModel(confDir, &cfg, "syncthing", Version, db)
 
 nextRepo:
 	for i, repo := range cfg.Repositories {
@@ -353,11 +363,14 @@ nextRepo:
 	// Walk the repository and update the local model before establishing any
 	// connections to other nodes.
 
-	l.Infoln("Populating repository index")
-	m.LoadIndexes(confDir)
+	if !havePersistentIndex {
+		// There's no new style index, load old ones
+		l.Infoln("Loading legacy index files")
+		m.LoadIndexes(confDir)
+	}
 	m.CleanRepos()
+	l.Infoln("Performing initial repository scan")
 	m.ScanRepos()
-	m.SaveIndexes(confDir)
 
 	// Remove all .idx* files that don't belong to an active repo.
 
@@ -709,7 +722,7 @@ next:
 				}
 				protoConn := protocol.NewConnection(remoteID, conn, wr, m)
 
-				l.Infof("Connection to %s established at %v", remoteID, conn.RemoteAddr())
+				l.Infof("Established secure connection to %s at %v", remoteID, conn.RemoteAddr())
 
 				m.AddConnection(conn, protoConn)
 				continue next

files/cmd/pidx/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/calmh/syncthing/files"
+	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	repo := flag.String("repo", "default", "Repository ID")
+	node := flag.String("node", "", "Node ID (blank for global)")
+	flag.Parse()
+
+	db, err := leveldb.OpenFile(flag.Arg(0), nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fs := files.NewSet(*repo, db)
+
+	if *node == "" {
+		log.Printf("*** Global index for repo %q", *repo)
+		fs.WithGlobal(func(f scanner.File) bool {
+			fmt.Println(f)
+			fmt.Println("\t", fs.Availability(f.Name))
+			return true
+		})
+	} else {
+		n, err := protocol.NodeIDFromString(*node)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("*** Have index for repo %q node %q", *repo, n)
+		fs.WithHave(n, func(f scanner.File) bool {
+			fmt.Println(f)
+			return true
+		})
+	}
+}

files/leveldb.go

@@ -0,0 +1,563 @@
+package files
+
+import (
+	"bytes"
+	"sort"
+
+	"github.com/calmh/syncthing/lamport"
+	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const (
+	keyTypeNode = iota
+	keyTypeGlobal
+)
+
+type fileVersion struct {
+	version uint64
+	node    []byte
+}
+
+type versionList struct {
+	versions []fileVersion
+}
+
+type fileList []scanner.File
+
+func (l fileList) Len() int {
+	return len(l)
+}
+
+func (l fileList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l fileList) Less(a, b int) bool {
+	return l[a].Name < l[b].Name
+}
+
+type dbReader interface {
+	Get([]byte, *opt.ReadOptions) ([]byte, error)
+}
+
+type dbWriter interface {
+	Put([]byte, []byte)
+	Delete([]byte)
+}
+
+/*
+
+keyTypeNode (1 byte)
+    repository (64 bytes)
+        node (32 bytes)
+            name (variable size)
+            	|
+            	scanner.File
+
+keyTypeGlobal (1 byte)
+	repository (64 bytes)
+		name (variable size)
+			|
+			[]fileVersion (sorted)
+
+*/
+
+func nodeKey(repo, node, file []byte) []byte {
+	k := make([]byte, 1+64+32+len(file))
+	k[0] = keyTypeNode
+	copy(k[1:], []byte(repo))
+	copy(k[1+64:], node[:])
+	copy(k[1+64+32:], []byte(file))
+	return k
+}
+
+func globalKey(repo, file []byte) []byte {
+	k := make([]byte, 1+64+len(file))
+	k[0] = keyTypeGlobal
+	copy(k[1:], []byte(repo))
+	copy(k[1+64:], []byte(file))
+	return k
+}
+
+func nodeKeyName(key []byte) []byte {
+	return key[1+64+32:]
+}
+
+func globalKeyName(key []byte) []byte {
+	return key[1+64:]
+}
+
+type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool
+
+type fileIterator func(f scanner.File) bool
+
+func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, deleteFn deletionHandler) bool {
+	sort.Sort(fileList(fs)) // sort list on name, same as on disk
+
+	start := nodeKey(repo, node, nil)                            // before all repo/node files
+	limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
+
+	batch := new(leveldb.Batch)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	moreDb := dbi.Next()
+	fsi := 0
+	changed := false
+
+	for {
+		var newName, oldName []byte
+		moreFs := fsi < len(fs)
+
+		if !moreDb && !moreFs {
+			break
+		}
+
+		if !moreFs && deleteFn == nil {
+			// We don't have any more updated files to process and deletion
+			// has not been requested, so we can exit early
+			break
+		}
+
+		if moreFs {
+			newName = []byte(fs[fsi].Name)
+		}
+
+		if moreDb {
+			oldName = nodeKeyName(dbi.Key())
+		}
+
+		cmp := bytes.Compare(newName, oldName)
+
+		if debug {
+			l.Debugf("generic replace; repo=%q node=%x moreFs=%v moreDb=%v cmp=%d newName=%q oldName=%q", repo, node, moreFs, moreDb, cmp, newName, oldName)
+		}
+
+		switch {
+		case moreFs && (!moreDb || cmp == -1):
+			changed = true
+			// Disk is missing this file. Insert it.
+			ldbInsert(batch, repo, node, newName, fs[fsi])
+			ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+			fsi++
+
+		case cmp == 0:
+			// File exists on both sides - compare versions.
+			var ef scanner.File
+			ef.UnmarshalXDR(dbi.Value())
+			if fs[fsi].Version > ef.Version {
+				ldbInsert(batch, repo, node, newName, fs[fsi])
+				ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+				changed = true
+			}
+			// Iterate both sides.
+			fsi++
+			moreDb = dbi.Next()
+
+		case moreDb && (!moreFs || cmp == 1):
+			if deleteFn != nil {
+				if deleteFn(snap, batch, repo, node, oldName, dbi) {
+					changed = true
+				}
+			}
+			moreDb = dbi.Next()
+		}
+	}
+
+	err = db.Write(batch, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	return changed
+}
+
+func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
+		// Disk has files that we are missing. Remove it.
+		if debug {
+			l.Debugf("delete; repo=%q node=%x name=%q", repo, node, name)
+		}
+		batch.Delete(dbi.Key())
+		ldbRemoveFromGlobal(db, batch, repo, node, name)
+		return true
+	})
+}
+
+func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
+		var f scanner.File
+		err := f.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if !protocol.IsDeleted(f.Flags) {
+			if debug {
+				l.Debugf("mark deleted; repo=%q node=%x name=%q", repo, node, name)
+			}
+			f.Blocks = nil
+			f.Version = lamport.Default.Tick(f.Version)
+			f.Flags |= protocol.FlagDeleted
+			batch.Put(dbi.Key(), f.MarshalXDR())
+			ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
+			return true
+		}
+		return false
+	})
+}
+
+func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
+	batch := new(leveldb.Batch)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	for _, f := range fs {
+		name := []byte(f.Name)
+		fk := nodeKey(repo, node, name)
+		bs, err := snap.Get(fk, nil)
+		if err == leveldb.ErrNotFound {
+			ldbInsert(batch, repo, node, name, f)
+			ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			continue
+		}
+
+		var ef scanner.File
+		err = ef.UnmarshalXDR(bs)
+		if err != nil {
+			panic(err)
+		}
+		if ef.Version != f.Version {
+			ldbInsert(batch, repo, node, name, f)
+			ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+		}
+	}
+
+	err = db.Write(batch, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	return true
+}
+
+func ldbInsert(batch dbWriter, repo, node, name []byte, file scanner.File) {
+	if debug {
+		l.Debugf("insert; repo=%q node=%x %v", repo, node, file)
+	}
+
+	nk := nodeKey(repo, node, name)
+	batch.Put(nk, file.MarshalXDR())
+}
+
+// ldbUpdateGlobal adds this node+version to the version list for the given
+// file. If the node is already present in the list, the version is updated.
+// If the file does not have an entry in the global list, it is created.
+func ldbUpdateGlobal(db dbReader, batch dbWriter, repo, node, file []byte, version uint64) bool {
+	if debug {
+		l.Debugf("update global; repo=%q node=%x file=%q version=%d", repo, node, file, version)
+	}
+	gk := globalKey(repo, file)
+	svl, err := db.Get(gk, nil)
+	if err != nil && err != leveldb.ErrNotFound {
+		panic(err)
+	}
+
+	var fl versionList
+	nv := fileVersion{
+		node:    node,
+		version: version,
+	}
+	if svl != nil {
+		err = fl.UnmarshalXDR(svl)
+		if err != nil {
+			panic(err)
+		}
+
+		for i := range fl.versions {
+			if bytes.Compare(fl.versions[i].node, node) == 0 {
+				if fl.versions[i].version == version {
+					// No need to do anything
+					return false
+				}
+				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+				break
+			}
+		}
+	}
+
+	for i := range fl.versions {
+		if fl.versions[i].version <= version {
+			t := append(fl.versions, fileVersion{})
+			copy(t[i+1:], t[i:])
+			t[i] = nv
+			fl.versions = t
+			goto done
+		}
+	}
+
+	fl.versions = append(fl.versions, nv)
+
+done:
+	batch.Put(gk, fl.MarshalXDR())
+
+	return true
+}
+
+// ldbRemoveFromGlobal removes the node from the global version list for the
+// given file. If the version list is empty after this, the file entry is
+// removed entirely.
+func ldbRemoveFromGlobal(db dbReader, batch dbWriter, repo, node, file []byte) {
+	if debug {
+		l.Debugf("remove from global; repo=%q node=%x file=%q", repo, node, file)
+	}
+
+	gk := globalKey(repo, file)
+	svl, err := db.Get(gk, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	var fl versionList
+	err = fl.UnmarshalXDR(svl)
+	if err != nil {
+		panic(err)
+	}
+
+	for i := range fl.versions {
+		if bytes.Compare(fl.versions[i].node, node) == 0 {
+			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+			break
+		}
+	}
+
+	if len(fl.versions) == 0 {
+		batch.Delete(gk)
+	} else {
+		batch.Put(gk, fl.MarshalXDR())
+	}
+}
+
+func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
+	start := nodeKey(repo, node, nil)                            // before all repo/node files
+	limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		var f scanner.File
+		err := f.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if cont := fn(f); !cont {
+			return
+		}
+	}
+}
+
+func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File {
+	nk := nodeKey(repo, node, file)
+	bs, err := db.Get(nk, nil)
+	if err == leveldb.ErrNotFound {
+		return scanner.File{}
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var f scanner.File
+	err = f.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
+	k := globalKey(repo, file)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	bs, err := snap.Get(k, nil)
+	if err == leveldb.ErrNotFound {
+		return scanner.File{}
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var vl versionList
+	err = vl.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	if len(vl.versions) == 0 {
+		l.Debugln(k)
+		panic("no versions?")
+	}
+
+	k = nodeKey(repo, vl.versions[0].node, file)
+	bs, err = snap.Get(k, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	var f scanner.File
+	err = f.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) {
+	start := globalKey(repo, nil)
+	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		var vl versionList
+		err := vl.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if len(vl.versions) == 0 {
+			l.Debugln(dbi.Key())
+			panic("no versions?")
+		}
+		fk := nodeKey(repo, vl.versions[0].node, globalKeyName(dbi.Key()))
+		bs, err := snap.Get(fk, nil)
+		if err != nil {
+			panic(err)
+		}
+
+		var f scanner.File
+		err = f.UnmarshalXDR(bs)
+		if err != nil {
+			panic(err)
+		}
+
+		if cont := fn(f); !cont {
+			return
+		}
+	}
+}
+
+func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
+	k := globalKey(repo, file)
+	bs, err := db.Get(k, nil)
+	if err == leveldb.ErrNotFound {
+		return nil
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var vl versionList
+	err = vl.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+
+	var nodes []protocol.NodeID
+	for _, v := range vl.versions {
+		if v.version != vl.versions[0].version {
+			break
+		}
+		var n protocol.NodeID
+		copy(n[:], v.node)
+		nodes = append(nodes, n)
+	}
+
+	return nodes
+}
+
+func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
+	start := globalKey(repo, nil)
+	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		var vl versionList
+		err := vl.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if len(vl.versions) == 0 {
+			l.Debugln(dbi.Key())
+			panic("no versions?")
+		}
+
+		have := false // If we have the file, any version
+		need := false // If we have a lower version of the file
+		var haveVersion uint64
+		for _, v := range vl.versions {
+			if bytes.Compare(v.node, node) == 0 {
+				have = true
+				haveVersion = v.version
+				need = v.version < vl.versions[0].version
+				break
+			}
+		}
+
+		if need || !have {
+			name := globalKeyName(dbi.Key())
+			if debug {
+				l.Debugf("need repo=%q node=%x name=%q need=%v have=%v haveV=%d globalV=%d", repo, node, name, need, have, haveVersion, vl.versions[0].version)
+			}
+			fk := nodeKey(repo, vl.versions[0].node, name)
+			bs, err := snap.Get(fk, nil)
+			if err != nil {
+				panic(err)
+			}
+
+			var gf scanner.File
+			err = gf.UnmarshalXDR(bs)
+			if err != nil {
+				panic(err)
+			}
+
+			if protocol.IsDeleted(gf.Flags) && !have {
+				// We don't need deleted files that we don't have
+				continue
+			}
+
+			if cont := fn(gf); !cont {
+				return
+			}
+		}
+	}
+}

files/leveldb_xdr.go

@@ -0,0 +1,83 @@
+package files
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/syncthing/xdr"
+)
+
+func (o fileVersion) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o fileVersion) MarshalXDR() []byte {
+	var aw = make(xdr.AppendWriter, 0, 128)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o fileVersion) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(o.version)
+	xw.WriteBytes(o.node)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *fileVersion) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *fileVersion) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *fileVersion) decodeXDR(xr *xdr.Reader) error {
+	o.version = xr.ReadUint64()
+	o.node = xr.ReadBytes()
+	return xr.Error()
+}
+
+func (o versionList) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o versionList) MarshalXDR() []byte {
+	var aw = make(xdr.AppendWriter, 0, 128)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o versionList) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(uint32(len(o.versions)))
+	for i := range o.versions {
+		o.versions[i].encodeXDR(xw)
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *versionList) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *versionList) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *versionList) decodeXDR(xr *xdr.Reader) error {
+	_versionsSize := int(xr.ReadUint32())
+	o.versions = make([]fileVersion, _versionsSize)
+	for i := range o.versions {
+		(&o.versions[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}

files/set.go

@@ -8,10 +8,9 @@ package files
 import (
 	"sync"
 
-	"github.com/calmh/syncthing/cid"
-	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
+	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type fileRecord struct {
@@ -23,297 +22,97 @@ type fileRecord struct {
 type bitset uint64
 
 type Set struct {
-	sync.Mutex
-	files              map[key]fileRecord
-	remoteKey          [64]map[string]key
-	changes            [64]uint64
-	globalAvailability map[string]bitset
-	globalKey          map[string]key
+	changes map[protocol.NodeID]uint64
+	mutex   sync.RWMutex
+	repo    string
+	db      *leveldb.DB
 }
 
-func NewSet() *Set {
-	var m = Set{
-		files:              make(map[key]fileRecord),
-		globalAvailability: make(map[string]bitset),
-		globalKey:          make(map[string]key),
+func NewSet(repo string, db *leveldb.DB) *Set {
+	var s = Set{
+		changes: make(map[protocol.NodeID]uint64),
+		repo:    repo,
+		db:      db,
 	}
-	return &m
+	return &s
 }
 
-func (m *Set) Replace(id uint, fs []scanner.File) {
+func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) {
 	if debug {
-		l.Debugf("Replace(%d, [%d])", id, len(fs))
+		l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
 	}
-	if id > 63 {
-		panic("Connection ID must be in the range 0 - 63 inclusive")
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if ldbReplace(s.db, []byte(s.repo), node[:], fs) {
+		s.changes[node]++
 	}
-
-	m.Lock()
-	if len(fs) == 0 || !m.equals(id, fs) {
-		m.changes[id]++
-		m.replace(id, fs)
-	}
-	m.Unlock()
 }
 
-func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
+func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) {
 	if debug {
-		l.Debugf("ReplaceWithDelete(%d, [%d])", id, len(fs))
+		l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
 	}
-	if id > 63 {
-		panic("Connection ID must be in the range 0 - 63 inclusive")
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if ldbReplaceWithDelete(s.db, []byte(s.repo), node[:], fs) {
+		s.changes[node]++
 	}
-
-	m.Lock()
-	if len(fs) == 0 || !m.equals(id, fs) {
-		m.changes[id]++
-
-		var nf = make(map[string]key, len(fs))
-		for _, f := range fs {
-			nf[f.Name] = keyFor(f)
-		}
-
-		// For previously existing files not in the list, add them to the list
-		// with the relevant delete flags etc set. Previously existing files
-		// with the delete bit already set are not modified.
-
-		for _, ck := range m.remoteKey[cid.LocalID] {
-			if _, ok := nf[ck.Name]; !ok {
-				cf := m.files[ck].File
-				if !protocol.IsDeleted(cf.Flags) {
-					cf.Flags |= protocol.FlagDeleted
-					cf.Blocks = nil
-					cf.Size = 0
-					cf.Version = lamport.Default.Tick(cf.Version)
-				}
-				fs = append(fs, cf)
-				if debug {
-					l.Debugln("deleted:", ck.Name)
-				}
-			}
-		}
-
-		m.replace(id, fs)
-	}
-	m.Unlock()
 }
 
-func (m *Set) Update(id uint, fs []scanner.File) {
+func (s *Set) Update(node protocol.NodeID, fs []scanner.File) {
 	if debug {
-		l.Debugf("Update(%d, [%d])", id, len(fs))
+		l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
+	}
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if ldbUpdate(s.db, []byte(s.repo), node[:], fs) {
+		s.changes[node]++
 	}
-	m.Lock()
-	m.update(id, fs)
-	m.changes[id]++
-	m.Unlock()
 }
 
-func (m *Set) Need(id uint) []scanner.File {
+func (s *Set) WithNeed(node protocol.NodeID, fn fileIterator) {
 	if debug {
-		l.Debugf("Need(%d)", id)
+		l.Debugf("%s Need(%v)", s.repo, node)
 	}
-	m.Lock()
-	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
-	rkID := m.remoteKey[id]
-	for gk, gf := range m.files {
-		if !gf.Global || gf.File.Suppressed {
-			continue
-		}
-
-		if rk, ok := rkID[gk.Name]; gk.newerThan(rk) {
-			if protocol.IsDeleted(gf.File.Flags) && (!ok || protocol.IsDeleted(m.files[rk].File.Flags)) {
-				// We don't need to delete files we don't have or that are already deleted
-				continue
-			}
-
-			fs = append(fs, gf.File)
-		}
-	}
-	m.Unlock()
-	return fs
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	ldbWithNeed(s.db, []byte(s.repo), node[:], fn)
 }
 
-func (m *Set) Have(id uint) []scanner.File {
+func (s *Set) WithHave(node protocol.NodeID, fn fileIterator) {
 	if debug {
-		l.Debugf("Have(%d)", id)
+		l.Debugf("%s WithHave(%v)", s.repo, node)
 	}
-	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
-	m.Lock()
-	for _, rk := range m.remoteKey[id] {
-		fs = append(fs, m.files[rk].File)
-	}
-	m.Unlock()
-	return fs
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	ldbWithHave(s.db, []byte(s.repo), node[:], fn)
 }
 
-func (m *Set) Global() []scanner.File {
+func (s *Set) WithGlobal(fn fileIterator) {
 	if debug {
-		l.Debugf("Global()")
+		l.Debugf("%s WithGlobal()", s.repo)
 	}
-	m.Lock()
-	var fs = make([]scanner.File, 0, len(m.globalKey))
-	for _, file := range m.files {
-		if file.Global {
-			fs = append(fs, file.File)
-		}
-	}
-	m.Unlock()
-	return fs
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	ldbWithGlobal(s.db, []byte(s.repo), fn)
 }
 
-func (m *Set) Get(id uint, file string) scanner.File {
-	m.Lock()
-	defer m.Unlock()
-	if debug {
-		l.Debugf("Get(%d, %q)", id, file)
-	}
-	return m.files[m.remoteKey[id][file]].File
+func (s *Set) Get(node protocol.NodeID, file string) scanner.File {
+	return ldbGet(s.db, []byte(s.repo), node[:], []byte(file))
 }
 
-func (m *Set) GetGlobal(file string) scanner.File {
-	m.Lock()
-	defer m.Unlock()
-	if debug {
-		l.Debugf("GetGlobal(%q)", file)
-	}
-	return m.files[m.globalKey[file]].File
+func (s *Set) GetGlobal(file string) scanner.File {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return ldbGetGlobal(s.db, []byte(s.repo), []byte(file))
 }
 
-func (m *Set) Availability(name string) bitset {
-	m.Lock()
-	defer m.Unlock()
-	av := m.globalAvailability[name]
-	if debug {
-		l.Debugf("Availability(%q) = %0x", name, av)
-	}
-	return av
+func (s *Set) Availability(file string) []protocol.NodeID {
+	return ldbAvailability(s.db, []byte(s.repo), []byte(file))
 }
 
-func (m *Set) Changes(id uint) uint64 {
-	m.Lock()
-	defer m.Unlock()
-	if debug {
-		l.Debugf("Changes(%d)", id)
-	}
-	return m.changes[id]
-}
-
-func (m *Set) equals(id uint, fs []scanner.File) bool {
-	curWithoutDeleted := make(map[string]key)
-	for _, k := range m.remoteKey[id] {
-		f := m.files[k].File
-		if !protocol.IsDeleted(f.Flags) {
-			curWithoutDeleted[f.Name] = k
-		}
-	}
-	if len(curWithoutDeleted) != len(fs) {
-		return false
-	}
-	for _, f := range fs {
-		if curWithoutDeleted[f.Name] != keyFor(f) {
-			return false
-		}
-	}
-	return true
-}
-
-func (m *Set) update(cid uint, fs []scanner.File) {
-	remFiles := m.remoteKey[cid]
-	if remFiles == nil {
-		l.Fatalln("update before replace for cid", cid)
-	}
-	for _, f := range fs {
-		n := f.Name
-		fk := keyFor(f)
-
-		if ck, ok := remFiles[n]; ok && ck == fk {
-			// The remote already has exactly this file, skip it
-			continue
-		}
-
-		remFiles[n] = fk
-
-		// Keep the block list or increment the usage
-		if br, ok := m.files[fk]; !ok {
-			m.files[fk] = fileRecord{
-				Usage: 1,
-				File:  f,
-			}
-		} else {
-			br.Usage++
-			m.files[fk] = br
-		}
-
-		// Update global view
-		gk, ok := m.globalKey[n]
-		switch {
-		case ok && fk == gk:
-			av := m.globalAvailability[n]
-			av |= 1 << cid
-			m.globalAvailability[n] = av
-		case fk.newerThan(gk):
-			if ok {
-				f := m.files[gk]
-				f.Global = false
-				m.files[gk] = f
-			}
-			f := m.files[fk]
-			f.Global = true
-			m.files[fk] = f
-			m.globalKey[n] = fk
-			m.globalAvailability[n] = 1 << cid
-		}
-	}
-}
-
-func (m *Set) replace(cid uint, fs []scanner.File) {
-	// Decrement usage for all files belonging to this remote, and remove
-	// those that are no longer needed.
-	for _, fk := range m.remoteKey[cid] {
-		br, ok := m.files[fk]
-		switch {
-		case ok && br.Usage == 1:
-			delete(m.files, fk)
-		case ok && br.Usage > 1:
-			br.Usage--
-			m.files[fk] = br
-		}
-	}
-
-	// Clear existing remote remoteKey
-	m.remoteKey[cid] = make(map[string]key)
-
-	// Recalculate global based on all remaining remoteKey
-	for n := range m.globalKey {
-		var nk key    // newest key
-		var na bitset // newest availability
-
-		for i, rem := range m.remoteKey {
-			if rk, ok := rem[n]; ok {
-				switch {
-				case rk == nk:
-					na |= 1 << uint(i)
-				case rk.newerThan(nk):
-					nk = rk
-					na = 1 << uint(i)
-				}
-			}
-		}
-
-		if na != 0 {
-			// Someone had the file
-			f := m.files[nk]
-			f.Global = true
-			m.files[nk] = f
-			m.globalKey[n] = nk
-			m.globalAvailability[n] = na
-		} else {
-			// Noone had the file
-			delete(m.globalKey, n)
-			delete(m.globalAvailability, n)
-		}
-	}
-
-	// Add new remote remoteKey to the mix
-	m.update(cid, fs)
+func (s *Set) Changes(node protocol.NodeID) uint64 {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return s.changes[node]
 }

files/set_anal.go

@@ -1,48 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-// Use of this source code is governed by an MIT-style license that can be
-// found in the LICENSE file.
-
-//+build anal
-
-package files
-
-import (
-	"crypto/md5"
-
-	"github.com/calmh/syncthing/scanner"
-)
-
-type key struct {
-	Name     string
-	Version  uint64
-	Modified int64
-	Hash     [md5.Size]byte
-}
-
-func keyFor(f scanner.File) key {
-	h := md5.New()
-	for _, b := range f.Blocks {
-		h.Write(b.Hash)
-	}
-	return key{
-		Name:     f.Name,
-		Version:  f.Version,
-		Modified: f.Modified,
-		Hash:     md5.Sum(nil),
-	}
-}
-
-func (a key) newerThan(b key) bool {
-	if a.Version != b.Version {
-		return a.Version > b.Version
-	}
-	if a.Modified != b.Modified {
-		return a.Modified > b.Modified
-	}
-	for i := 0; i < md5.Size; i++ {
-		if a.Hash[i] != b.Hash[i] {
-			return a.Hash[i] > b.Hash[i]
-		}
-	}
-	return false
-}

files/set_fast.go

@@ -1,25 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-// Use of this source code is governed by an MIT-style license that can be
-// found in the LICENSE file.
-
-//+build !anal
-
-package files
-
-import "github.com/calmh/syncthing/scanner"
-
-type key struct {
-	Name    string
-	Version uint64
-}
-
-func keyFor(f scanner.File) key {
-	return key{
-		Name:    f.Name,
-		Version: f.Version,
-	}
-}
-
-func (a key) newerThan(b key) bool {
-	return a.Version > b.Version
-}

files/set_test.go

@@ -6,17 +6,64 @@ package files_test
 
 import (
 	"fmt"
-	"reflect"
 	"sort"
 	"testing"
 
-	"github.com/calmh/syncthing/cid"
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
+var remoteNode protocol.NodeID
+
+func init() {
+	remoteNode, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
+}
+
+func genBlocks(n int) []scanner.Block {
+	b := make([]scanner.Block, n)
+	for i := range b {
+		h := make([]byte, 32)
+		for j := range h {
+			h[j] = byte(i + j)
+		}
+		b[i].Size = uint32(i)
+		b[i].Offset = int64(i)
+		b[i].Hash = h
+	}
+	return b
+}
+
+func globalList(s *files.Set) []scanner.File {
+	var fs []scanner.File
+	s.WithGlobal(func(f scanner.File) bool {
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func haveList(s *files.Set, n protocol.NodeID) []scanner.File {
+	var fs []scanner.File
+	s.WithHave(n, func(f scanner.File) bool {
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func needList(s *files.Set, n protocol.NodeID) []scanner.File {
+	var fs []scanner.File
+	s.WithNeed(n, func(f scanner.File) bool {
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
 type fileList []scanner.File
 
 func (l fileList) Len() int {
@@ -32,118 +79,157 @@ func (l fileList) Swap(a, b int) {
 }
 
 func TestGlobalSet(t *testing.T) {
-	m := files.NewSet()
+	lamport.Default = lamport.Clock{}
 
-	local := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1000},
-		scanner.File{Name: "c", Version: 1000},
-		scanner.File{Name: "d", Version: 1000},
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	local0 := []scanner.File{
+		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+		scanner.File{Name: "z", Version: 1000, Blocks: genBlocks(8)},
+	}
+	local1 := []scanner.File{
+		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+	}
+	localTot := []scanner.File{
+		local0[0],
+		local0[1],
+		local0[2],
+		local0[3],
+		scanner.File{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
 	}
 
 	remote0 := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "c", Version: 1002},
+		scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		scanner.File{Name: "c", Version: 1002, Blocks: genBlocks(5)},
 	}
 	remote1 := []scanner.File{
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "e", Version: 1000},
+		scanner.File{Name: "b", Version: 1001, Blocks: genBlocks(6)},
+		scanner.File{Name: "e", Version: 1000, Blocks: genBlocks(7)},
 	}
 	remoteTot := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+		remote0[0],
+		remote1[0],
+		remote0[2],
+		remote1[1],
 	}
 
 	expectedGlobal := []scanner.File{
-		scanner.File{Name: "a", Version: 1000},
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "d", Version: 1000},
-		scanner.File{Name: "e", Version: 1000},
+		remote0[0],
+		remote1[0],
+		remote0[2],
+		localTot[3],
+		remote1[1],
+		localTot[4],
 	}
 
 	expectedLocalNeed := []scanner.File{
-		scanner.File{Name: "b", Version: 1001},
-		scanner.File{Name: "c", Version: 1002},
-		scanner.File{Name: "e", Version: 1000},
+		remote1[0],
+		remote0[2],
+		remote1[1],
 	}
 
 	expectedRemoteNeed := []scanner.File{
-		scanner.File{Name: "d", Version: 1000},
+		local0[3],
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
-	m.Replace(1, remote0)
-	m.Update(1, remote1)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local0)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
+	m.Replace(remoteNode, remote0)
+	m.Update(remoteNode, remote1)
 
-	g := m.Global()
+	g := globalList(m)
 	sort.Sort(fileList(g))
 
-	if !reflect.DeepEqual(g, expectedGlobal) {
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
 	}
 
-	h := m.Have(cid.LocalID)
+	h := haveList(m, protocol.LocalNodeID)
 	sort.Sort(fileList(h))
 
-	if !reflect.DeepEqual(h, local) {
-		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, local)
+	if fmt.Sprint(h) != fmt.Sprint(localTot) {
+		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
 	}
 
-	h = m.Have(1)
+	h = haveList(m, remoteNode)
 	sort.Sort(fileList(h))
 
-	if !reflect.DeepEqual(h, remoteTot) {
+	if fmt.Sprint(h) != fmt.Sprint(remoteTot) {
 		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, remoteTot)
 	}
 
-	n := m.Need(cid.LocalID)
+	n := needList(m, protocol.LocalNodeID)
 	sort.Sort(fileList(n))
 
-	if !reflect.DeepEqual(n, expectedLocalNeed) {
+	if fmt.Sprint(n) != fmt.Sprint(expectedLocalNeed) {
 		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedLocalNeed)
 	}
 
-	n = m.Need(1)
+	n = needList(m, remoteNode)
 	sort.Sort(fileList(n))
 
-	if !reflect.DeepEqual(n, expectedRemoteNeed) {
+	if fmt.Sprint(n) != fmt.Sprint(expectedRemoteNeed) {
 		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
 	}
 
-	f := m.Get(cid.LocalID, "b")
-	if !reflect.DeepEqual(f, local[1]) {
-		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, local[1])
+	f := m.Get(protocol.LocalNodeID, "b")
+	if fmt.Sprint(f) != fmt.Sprint(localTot[1]) {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
 	}
 
-	f = m.Get(1, "b")
-	if !reflect.DeepEqual(f, remote1[0]) {
+	f = m.Get(remoteNode, "b")
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
 		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
 	}
 
 	f = m.GetGlobal("b")
-	if !reflect.DeepEqual(f, remote1[0]) {
-		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
 	}
 
-	a := int(m.Availability("a"))
-	if av := 1<<0 + 1<<1; a != av {
+	f = m.Get(protocol.LocalNodeID, "zz")
+	if f.Name != "" {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, scanner.File{})
+	}
+
+	f = m.GetGlobal("zz")
+	if f.Name != "" {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, scanner.File{})
+	}
+
+	av := []protocol.NodeID{protocol.LocalNodeID, remoteNode}
+	a := m.Availability("a")
+	if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
 		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
 	}
-	a = int(m.Availability("b"))
-	if av := 1 << 1; a != av {
-		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
+	a = m.Availability("b")
+	if len(a) != 1 || a[0] != remoteNode {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteNode)
 	}
-	a = int(m.Availability("d"))
-	if av := 1 << 0; a != av {
-		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
+	a = m.Availability("d")
+	if len(a) != 1 || a[0] != protocol.LocalNodeID {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalNodeID)
 	}
 }
 
 func TestLocalDeleted(t *testing.T) {
-	m := files.NewSet()
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := files.NewSet("test", db)
 	lamport.Default = lamport.Clock{}
 
 	local1 := []scanner.File{
@@ -154,22 +240,22 @@ func TestLocalDeleted(t *testing.T) {
 		scanner.File{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local1)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
 
-	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
 		local1[0],
 		// [1] removed
 		local1[2],
 		local1[3],
 		local1[4],
 	})
-	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
 		local1[0],
 		local1[2],
 		// [3] removed
 		local1[4],
 	})
-	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
 		local1[0],
 		local1[2],
 		// [4] removed
@@ -183,15 +269,15 @@ func TestLocalDeleted(t *testing.T) {
 		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
-	g := m.Global()
+	g := globalList(m)
 	sort.Sort(fileList(g))
 	sort.Sort(fileList(expectedGlobal1))
 
-	if !reflect.DeepEqual(g, expectedGlobal1) {
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal1) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+	m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{
 		local1[0],
 		// [2] removed
 	})
@@ -204,16 +290,21 @@ func TestLocalDeleted(t *testing.T) {
 		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
-	g = m.Global()
+	g = globalList(m)
 	sort.Sort(fileList(g))
 	sort.Sort(fileList(expectedGlobal2))
 
-	if !reflect.DeepEqual(g, expectedGlobal2) {
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal2) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal2)
 	}
 }
 
 func Benchmark10kReplace(b *testing.B) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
 	var local []scanner.File
 	for i := 0; i < 10000; i++ {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
@@ -221,8 +312,8 @@ func Benchmark10kReplace(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m := files.NewSet()
-		m.ReplaceWithDelete(cid.LocalID, local)
+		m := files.NewSet("test", db)
+		m.ReplaceWithDelete(protocol.LocalNodeID, local)
 	}
 }
 
@@ -232,15 +323,20 @@ func Benchmark10kUpdateChg(b *testing.B) {
 		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m := files.NewSet()
-	m.Replace(1, remote)
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode, remote)
 
 	var local []scanner.File
 	for i := 0; i < 10000; i++ {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -249,7 +345,7 @@ func Benchmark10kUpdateChg(b *testing.B) {
 			local[j].Version++
 		}
 		b.StartTimer()
-		m.Update(cid.LocalID, local)
+		m.Update(protocol.LocalNodeID, local)
 	}
 }
 
@@ -259,19 +355,23 @@ func Benchmark10kUpdateSme(b *testing.B) {
 		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m := files.NewSet()
-	m.Replace(1, remote)
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode, remote)
 
 	var local []scanner.File
 	for i := 0; i < 10000; i++ {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Update(cid.LocalID, local)
+		m.Update(protocol.LocalNodeID, local)
 	}
 }
 
@@ -281,8 +381,13 @@ func Benchmark10kNeed2k(b *testing.B) {
 		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m := files.NewSet()
-	m.Replace(cid.LocalID+1, remote)
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode, remote)
 
 	var local []scanner.File
 	for i := 0; i < 8000; i++ {
@@ -292,25 +397,30 @@ func Benchmark10kNeed2k(b *testing.B) {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		fs := m.Need(cid.LocalID)
+		fs := needList(m, protocol.LocalNodeID)
 		if l := len(fs); l != 2000 {
 			b.Errorf("wrong length %d != 2k", l)
 		}
 	}
 }
 
-func Benchmark10kHave(b *testing.B) {
+func Benchmark10kHaveFullList(b *testing.B) {
 	var remote []scanner.File
 	for i := 0; i < 10000; i++ {
 		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m := files.NewSet()
-	m.Replace(cid.LocalID+1, remote)
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode, remote)
 
 	var local []scanner.File
 	for i := 0; i < 2000; i++ {
@@ -320,11 +430,11 @@ func Benchmark10kHave(b *testing.B) {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		fs := m.Have(cid.LocalID)
+		fs := haveList(m, protocol.LocalNodeID)
 		if l := len(fs); l != 10000 {
 			b.Errorf("wrong length %d != 10k", l)
 		}
@@ -337,8 +447,13 @@ func Benchmark10kGlobal(b *testing.B) {
 		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m := files.NewSet()
-	m.Replace(cid.LocalID+1, remote)
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode, remote)
 
 	var local []scanner.File
 	for i := 0; i < 2000; i++ {
@@ -348,11 +463,11 @@ func Benchmark10kGlobal(b *testing.B) {
 		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		fs := m.Global()
+		fs := globalList(m)
 		if l := len(fs); l != 10000 {
 			b.Errorf("wrong length %d != 10k", l)
 		}
@@ -360,7 +475,12 @@ func Benchmark10kGlobal(b *testing.B) {
 }
 
 func TestGlobalReset(t *testing.T) {
-	m := files.NewSet()
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
 
 	local := []scanner.File{
 		scanner.File{Name: "a", Version: 1000},
@@ -376,27 +496,32 @@ func TestGlobalReset(t *testing.T) {
 		scanner.File{Name: "e", Version: 1000},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
-	g := m.Global()
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+	g := globalList(m)
 	sort.Sort(fileList(g))
 
-	if !reflect.DeepEqual(g, local) {
+	if fmt.Sprint(g) != fmt.Sprint(local) {
 		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
 	}
 
-	m.Replace(1, remote)
-	m.Replace(1, nil)
+	m.Replace(remoteNode, remote)
+	m.Replace(remoteNode, nil)
 
-	g = m.Global()
+	g = globalList(m)
 	sort.Sort(fileList(g))
 
-	if !reflect.DeepEqual(g, local) {
+	if fmt.Sprint(g) != fmt.Sprint(local) {
 		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
 	}
 }
 
 func TestNeed(t *testing.T) {
-	m := files.NewSet()
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
 
 	local := []scanner.File{
 		scanner.File{Name: "a", Version: 1000},
@@ -418,21 +543,26 @@ func TestNeed(t *testing.T) {
 		scanner.File{Name: "e", Version: 1000},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local)
-	m.Replace(1, remote)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+	m.Replace(remoteNode, remote)
 
-	need := m.Need(0)
+	need := needList(m, protocol.LocalNodeID)
 
 	sort.Sort(fileList(need))
 	sort.Sort(fileList(shouldNeed))
 
-	if !reflect.DeepEqual(need, shouldNeed) {
+	if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
 		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
 	}
 }
 
 func TestChanges(t *testing.T) {
-	m := files.NewSet()
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
 
 	local1 := []scanner.File{
 		scanner.File{Name: "a", Version: 1000},
@@ -449,17 +579,17 @@ func TestChanges(t *testing.T) {
 		scanner.File{Name: "e", Version: 1000},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local1)
-	c0 := m.Changes(cid.LocalID)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
+	c0 := m.Changes(protocol.LocalNodeID)
 
-	m.ReplaceWithDelete(cid.LocalID, local2)
-	c1 := m.Changes(cid.LocalID)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local2)
+	c1 := m.Changes(protocol.LocalNodeID)
 	if !(c1 > c0) {
 		t.Fatal("Change number should have incremented")
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local2)
-	c2 := m.Changes(cid.LocalID)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local2)
+	c2 := m.Changes(protocol.LocalNodeID)
 	if c2 != c1 {
 		t.Fatal("Change number should be unchanged")
 	}

files/testdata/.gitignore

@@ -0,0 +1 @@
+index.db

integration/.gitignore

@@ -15,3 +15,4 @@ dirs-*
 csrftokens.txt
 s4d
 http
+idx