From ef236228fa4e161aa92294e3f13ec6ffae802bbe Mon Sep 17 00:00:00 2001
From: miao <362622365@qq.com>
Date: Sun, 25 Sep 2016 00:55:55 +0800
Subject: [PATCH 01/59] Avoid allocate a hole page, when the node size equals
 to pageSize

---
 node.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node.go b/node.go
index 159318b2..f4ce240e 100644
--- a/node.go
+++ b/node.go
@@ -365,7 +365,7 @@ func (n *node) spill() error {
 		}
 
 		// Allocate contiguous space for the node.
-		p, err := tx.allocate((node.size() / tx.db.pageSize) + 1)
+		p, err := tx.allocate((node.size() + tx.db.pageSize - 1) / tx.db.pageSize)
 		if err != nil {
 			return err
 		}

From 78d099ed1ffce8e9fa4a65f92ae8d1fe6a0a08e8 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Sun, 21 May 2017 10:16:20 -0700
Subject: [PATCH 02/59] Garbage collect pages allocated after minimum txid

Read txns would lock pages allocated after the txn, keeping those pages
off the free list until closing the read txn. Instead, track allocating
txid to compute page lifetime, freeing pages if all txns between
page allocation and page free are closed.
---
 db.go            |  36 ++++++++++----
 freelist.go      | 121 +++++++++++++++++++++++++++++++++++++----------
 freelist_test.go |  36 +++++++-------
 tx.go            |   2 +-
 4 files changed, 143 insertions(+), 52 deletions(-)

diff --git a/db.go b/db.go
index f352ff14..17b14a5a 100644
--- a/db.go
+++ b/db.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"runtime"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -526,21 +527,36 @@ func (db *DB) beginRWTx() (*Tx, error) {
 	t := &Tx{writable: true}
 	t.init(db)
 	db.rwtx = t
+	db.freePages()
+	return t, nil
+}
 
-	// Free any pages associated with closed read-only transactions.
-	var minid txid = 0xFFFFFFFFFFFFFFFF
-	for _, t := range db.txs {
-		if t.meta.txid < minid {
-			minid = t.meta.txid
-		}
+// freePages releases any pages associated with closed read-only transactions.
+func (db *DB) freePages() {
+	// Free all pending pages prior to earliest open transaction.
+	sort.Sort(txsById(db.txs))
+	minid := txid(0xFFFFFFFFFFFFFFFF)
+	if len(db.txs) > 0 {
+		minid = db.txs[0].meta.txid
 	}
 	if minid > 0 {
 		db.freelist.release(minid - 1)
 	}
-
-	return t, nil
+	// Release unused txid extents.
+	for _, t := range db.txs {
+		db.freelist.releaseRange(minid, t.meta.txid-1)
+		minid = t.meta.txid + 1
+	}
+	db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF))
+	// Any page both allocated and freed in an extent is safe to release.
 }
 
+type txsById []*Tx
+
+func (t txsById) Len() int           { return len(t) }
+func (t txsById) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
+func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid }
+
 // removeTx removes a transaction from the database.
 func (db *DB) removeTx(tx *Tx) {
 	// Release the read lock on the mmap.
@@ -826,7 +842,7 @@ func (db *DB) meta() *meta {
 }
 
 // allocate returns a contiguous block of memory starting at a given page.
-func (db *DB) allocate(count int) (*page, error) {
+func (db *DB) allocate(txid txid, count int) (*page, error) {
 	// Allocate a temporary buffer for the page.
 	var buf []byte
 	if count == 1 {
@@ -838,7 +854,7 @@ func (db *DB) allocate(count int) (*page, error) {
 	p.overflow = uint32(count - 1)
 
 	// Use pages from the freelist if they are available.
-	if p.id = db.freelist.allocate(count); p.id != 0 {
+	if p.id = db.freelist.allocate(txid, count); p.id != 0 {
 		return p, nil
 	}
 
diff --git a/freelist.go b/freelist.go
index aba48f58..e0892a6c 100644
--- a/freelist.go
+++ b/freelist.go
@@ -6,18 +6,29 @@ import (
 	"unsafe"
 )
 
+
+// txPending holds a list of pgids and corresponding allocation txns
+// that are pending to be freed.
+type txPending struct {
+	ids              []pgid
+	alloctx          []txid // txids allocating the ids
+	lastReleaseBegin txid   // beginning txid of last matching releaseRange
+}
+
 // freelist represents a list of all pages that are available for allocation.
 // It also tracks pages that have been freed but are still in use by open transactions.
 type freelist struct {
-	ids     []pgid          // all free and available free page ids.
-	pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
-	cache   map[pgid]bool   // fast lookup of all free and pending page ids.
+	ids     []pgid              // all free and available free page ids.
+	allocs  map[pgid]txid       // mapping of txid that allocated a pgid.
+	pending map[txid]*txPending // mapping of soon-to-be free page ids by tx.
+	cache   map[pgid]bool       // fast lookup of all free and pending page ids.
 }
 
 // newFreelist returns an empty, initialized freelist.
 func newFreelist() *freelist {
 	return &freelist{
-		pending: make(map[txid][]pgid),
+		allocs:  make(map[pgid]txid),
+		pending: make(map[txid]*txPending),
 		cache:   make(map[pgid]bool),
 	}
 }
@@ -45,8 +56,8 @@ func (f *freelist) free_count() int {
 // pending_count returns count of pending pages
 func (f *freelist) pending_count() int {
 	var count int
-	for _, list := range f.pending {
-		count += len(list)
+	for _, txp := range f.pending {
+		count += len(txp.ids)
 	}
 	return count
 }
@@ -55,8 +66,8 @@ func (f *freelist) pending_count() int {
 // f.count returns the minimum length required for dst.
 func (f *freelist) copyall(dst []pgid) {
 	m := make(pgids, 0, f.pending_count())
-	for _, list := range f.pending {
-		m = append(m, list...)
+	for _, txp := range f.pending {
+		m = append(m, txp.ids...)
 	}
 	sort.Sort(m)
 	mergepgids(dst, f.ids, m)
@@ -64,7 +75,7 @@ func (f *freelist) copyall(dst []pgid) {
 
 // allocate returns the starting page id of a contiguous list of pages of a given size.
 // If a contiguous block cannot be found then 0 is returned.
-func (f *freelist) allocate(n int) pgid {
+func (f *freelist) allocate(txid txid, n int) pgid {
 	if len(f.ids) == 0 {
 		return 0
 	}
@@ -97,7 +108,7 @@ func (f *freelist) allocate(n int) pgid {
 			for i := pgid(0); i < pgid(n); i++ {
 				delete(f.cache, initial+i)
 			}
-
+			f.allocs[initial] = txid
 			return initial
 		}
 
@@ -114,28 +125,73 @@ func (f *freelist) free(txid txid, p *page) {
 	}
 
 	// Free page and all its overflow pages.
-	var ids = f.pending[txid]
+	txp := f.pending[txid]
+	if txp == nil {
+		txp = &txPending{}
+		f.pending[txid] = txp
+	}
+	allocTxid, ok := f.allocs[p.id]
+	if ok {
+		delete(f.allocs, p.id)
+	} else if (p.flags & (freelistPageFlag | metaPageFlag)) != 0 {
+		// Safe to claim txid as allocating since these types are private to txid.
+		allocTxid = txid
+	}
+
 	for id := p.id; id <= p.id+pgid(p.overflow); id++ {
 		// Verify that page is not already free.
 		if f.cache[id] {
 			panic(fmt.Sprintf("page %d already freed", id))
 		}
-
 		// Add to the freelist and cache.
-		ids = append(ids, id)
+		txp.ids = append(txp.ids, id)
+		txp.alloctx = append(txp.alloctx, allocTxid)
 		f.cache[id] = true
 	}
-	f.pending[txid] = ids
 }
 
 // release moves all page ids for a transaction id (or older) to the freelist.
 func (f *freelist) release(txid txid) {
 	m := make(pgids, 0)
-	for tid, ids := range f.pending {
+	for tid, txp := range f.pending {
 		if tid <= txid {
 			// Move transaction's pending pages to the available freelist.
 			// Don't remove from the cache since the page is still free.
-			m = append(m, ids...)
+			m = append(m, txp.ids...)
+			delete(f.pending, tid)
+		}
+	}
+	sort.Sort(m)
+	f.ids = pgids(f.ids).merge(m)
+}
+
+// releaseRange moves pending pages allocated within an extent [begin,end] to the free list.
+func (f *freelist) releaseRange(begin, end txid) {
+	if begin > end {
+		return
+	}
+	var m pgids
+	for tid, txp := range f.pending {
+		if tid < begin || tid > end {
+			continue
+		}
+		// Don't recompute freed pages if ranges haven't updated.
+		if txp.lastReleaseBegin == begin {
+			continue
+		}
+		for i := 0; i < len(txp.ids); i++ {
+			if atx := txp.alloctx[i]; atx < begin || atx > end {
+				continue
+			}
+			m = append(m, txp.ids[i])
+			txp.ids[i] = txp.ids[len(txp.ids)-1]
+			txp.ids = txp.ids[:len(txp.ids)-1]
+			txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1]
+			txp.alloctx = txp.alloctx[:len(txp.alloctx)-1]
+			i--
+		}
+		txp.lastReleaseBegin = begin
+		if len(txp.ids) == 0 {
 			delete(f.pending, tid)
 		}
 	}
@@ -146,12 +202,29 @@ func (f *freelist) release(txid txid) {
 // rollback removes the pages from a given pending tx.
 func (f *freelist) rollback(txid txid) {
 	// Remove page ids from cache.
-	for _, id := range f.pending[txid] {
-		delete(f.cache, id)
+	txp := f.pending[txid]
+	if txp == nil {
+		return
 	}
-
-	// Remove pages from pending list.
+	var m pgids
+	for i, pgid := range txp.ids {
+		delete(f.cache, pgid)
+		tx := txp.alloctx[i]
+		if tx == 0 {
+			continue
+		}
+		if tx != txid {
+			// Pending free aborted; restore page back to alloc list.
+			f.allocs[pgid] = tx
+		} else {
+			// Freed page was allocated by this txn; OK to throw away.
+			m = append(m, pgid)
+		}
+	}
+	// Remove pages from pending list and mark as free if allocated by txid.
 	delete(f.pending, txid)
+	sort.Sort(m)
+	f.ids = pgids(f.ids).merge(m)
 }
 
 // freed returns whether a given page is in the free list.
@@ -217,8 +290,8 @@ func (f *freelist) reload(p *page) {
 
 	// Build a cache of only pending pages.
 	pcache := make(map[pgid]bool)
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			pcache[pendingID] = true
 		}
 	}
@@ -244,8 +317,8 @@ func (f *freelist) reindex() {
 	for _, id := range f.ids {
 		f.cache[id] = true
 	}
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			f.cache[pendingID] = true
 		}
 	}
diff --git a/freelist_test.go b/freelist_test.go
index 4e9b3a8d..4259bedc 100644
--- a/freelist_test.go
+++ b/freelist_test.go
@@ -12,7 +12,7 @@ import (
 func TestFreelist_free(t *testing.T) {
 	f := newFreelist()
 	f.free(100, &page{id: 12})
-	if !reflect.DeepEqual([]pgid{12}, f.pending[100]) {
+	if !reflect.DeepEqual([]pgid{12}, f.pending[100].ids) {
 		t.Fatalf("exp=%v; got=%v", []pgid{12}, f.pending[100])
 	}
 }
@@ -21,7 +21,7 @@ func TestFreelist_free(t *testing.T) {
 func TestFreelist_free_overflow(t *testing.T) {
 	f := newFreelist()
 	f.free(100, &page{id: 12, overflow: 3})
-	if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100]) {
+	if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100].ids) {
 		t.Fatalf("exp=%v; got=%v", exp, f.pending[100])
 	}
 }
@@ -46,39 +46,40 @@ func TestFreelist_release(t *testing.T) {
 
 // Ensure that a freelist can find contiguous blocks of pages.
 func TestFreelist_allocate(t *testing.T) {
-	f := &freelist{ids: []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}}
-	if id := int(f.allocate(3)); id != 3 {
+	f := newFreelist()
+	f.ids = []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}
+	if id := int(f.allocate(1, 3)); id != 3 {
 		t.Fatalf("exp=3; got=%v", id)
 	}
-	if id := int(f.allocate(1)); id != 6 {
+	if id := int(f.allocate(1, 1)); id != 6 {
 		t.Fatalf("exp=6; got=%v", id)
 	}
-	if id := int(f.allocate(3)); id != 0 {
+	if id := int(f.allocate(1, 3)); id != 0 {
 		t.Fatalf("exp=0; got=%v", id)
 	}
-	if id := int(f.allocate(2)); id != 12 {
+	if id := int(f.allocate(1, 2)); id != 12 {
 		t.Fatalf("exp=12; got=%v", id)
 	}
-	if id := int(f.allocate(1)); id != 7 {
+	if id := int(f.allocate(1, 1)); id != 7 {
 		t.Fatalf("exp=7; got=%v", id)
 	}
-	if id := int(f.allocate(0)); id != 0 {
+	if id := int(f.allocate(1, 0)); id != 0 {
 		t.Fatalf("exp=0; got=%v", id)
 	}
-	if id := int(f.allocate(0)); id != 0 {
+	if id := int(f.allocate(1, 0)); id != 0 {
 		t.Fatalf("exp=0; got=%v", id)
 	}
 	if exp := []pgid{9, 18}; !reflect.DeepEqual(exp, f.ids) {
 		t.Fatalf("exp=%v; got=%v", exp, f.ids)
 	}
 
-	if id := int(f.allocate(1)); id != 9 {
+	if id := int(f.allocate(1, 1)); id != 9 {
 		t.Fatalf("exp=9; got=%v", id)
 	}
-	if id := int(f.allocate(1)); id != 18 {
+	if id := int(f.allocate(1, 1)); id != 18 {
 		t.Fatalf("exp=18; got=%v", id)
 	}
-	if id := int(f.allocate(1)); id != 0 {
+	if id := int(f.allocate(1, 1)); id != 0 {
 		t.Fatalf("exp=0; got=%v", id)
 	}
 	if exp := []pgid{}; !reflect.DeepEqual(exp, f.ids) {
@@ -113,9 +114,9 @@ func TestFreelist_read(t *testing.T) {
 func TestFreelist_write(t *testing.T) {
 	// Create a freelist and write it to a page.
 	var buf [4096]byte
-	f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid][]pgid)}
-	f.pending[100] = []pgid{28, 11}
-	f.pending[101] = []pgid{3}
+	f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid]*txPending)}
+	f.pending[100] = &txPending{ids: []pgid{28, 11}}
+	f.pending[101] = &txPending{ids: []pgid{3}}
 	p := (*page)(unsafe.Pointer(&buf[0]))
 	if err := f.write(p); err != nil {
 		t.Fatal(err)
@@ -142,7 +143,8 @@ func benchmark_FreelistRelease(b *testing.B, size int) {
 	pending := randomPgids(len(ids) / 400)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		f := &freelist{ids: ids, pending: map[txid][]pgid{1: pending}}
+		txp := &txPending{ids: pending}
+		f := &freelist{ids: ids, pending: map[txid]*txPending{1: txp}}
 		f.release(1)
 	}
 }
diff --git a/tx.go b/tx.go
index 6700308a..f8cae6b1 100644
--- a/tx.go
+++ b/tx.go
@@ -453,7 +453,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo
 
 // allocate returns a contiguous block of memory starting at a given page.
 func (tx *Tx) allocate(count int) (*page, error) {
-	p, err := tx.db.allocate(count)
+	p, err := tx.db.allocate(tx.meta.txid, count)
 	if err != nil {
 		return nil, err
 	}

From 78b54a42e16e16138347b6b14fe74cb13479e271 Mon Sep 17 00:00:00 2001
From: Xiang <xiangli.cs@gmail.com>
Date: Tue, 20 Jun 2017 11:03:31 -0700
Subject: [PATCH 03/59] *: use coreos/bbolt as import path

---
 bucket_test.go        | 2 +-
 cmd/bolt/main_test.go | 4 ++--
 cursor_test.go        | 2 +-
 db_test.go            | 2 +-
 simulation_test.go    | 2 +-
 tx_test.go            | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/bucket_test.go b/bucket_test.go
index cddbe271..bcdd1dce 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -13,7 +13,7 @@ import (
 	"testing"
 	"testing/quick"
 
-	"github.com/boltdb/bolt"
+	"github.com/coreos/bbolt"
 )
 
 // Ensure that a bucket that gets a non-existent key returns nil.
diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go
index 0a11ff33..e3a834d2 100644
--- a/cmd/bolt/main_test.go
+++ b/cmd/bolt/main_test.go
@@ -12,8 +12,8 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/boltdb/bolt"
-	"github.com/boltdb/bolt/cmd/bolt"
+	"github.com/coreos/bbolt"
+	"github.com/coreos/bbolt/cmd/bolt"
 )
 
 // Ensure the "info" command can print information about a database.
diff --git a/cursor_test.go b/cursor_test.go
index 562d60f9..7b1ae198 100644
--- a/cursor_test.go
+++ b/cursor_test.go
@@ -11,7 +11,7 @@ import (
 	"testing"
 	"testing/quick"
 
-	"github.com/boltdb/bolt"
+	"github.com/coreos/bbolt"
 )
 
 // Ensure that a cursor can return a reference to the bucket that created it.
diff --git a/db_test.go b/db_test.go
index 3034d4f4..ad0e3d47 100644
--- a/db_test.go
+++ b/db_test.go
@@ -19,7 +19,7 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/boltdb/bolt"
+	"github.com/coreos/bbolt"
 )
 
 var statsFlag = flag.Bool("stats", false, "show performance stats")
diff --git a/simulation_test.go b/simulation_test.go
index 38310165..e6241530 100644
--- a/simulation_test.go
+++ b/simulation_test.go
@@ -7,7 +7,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/boltdb/bolt"
+	"github.com/coreos/bbolt"
 )
 
 func TestSimulate_1op_1p(t *testing.T)     { testSimulate(t, 1, 1) }
diff --git a/tx_test.go b/tx_test.go
index 2201e792..ff512b19 100644
--- a/tx_test.go
+++ b/tx_test.go
@@ -8,7 +8,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/boltdb/bolt"
+	"github.com/coreos/bbolt"
 )
 
 // Ensure that committing a closed transaction returns an error.

From 7149270521355d7b9369ac4cb4db4d849010066c Mon Sep 17 00:00:00 2001
From: Xiang <xiangli.cs@gmail.com>
Date: Thu, 22 Jun 2017 12:46:56 -0700
Subject: [PATCH 04/59] *: add option to skip freelist sync

When the database has a lot of freepages, the cost to sync all
freepages down to disk is high. If the total database size is
small (<10GB), and the application can tolerate ~10 seconds
recovery time, then it is reasonable to simply not sync freelist
and rescan the db to rebuild freelist on recovery.
---
 db.go                               |  53 +++++++++-
 db_test.go                          |  33 +++++-
 freelist.go                         |   6 ++
 simulation_no_freelist_sync_test.go |  47 +++++++++
 simulation_test.go                  | 151 +++++++++++++++-------------
 tx.go                               |  56 +++++++----
 6 files changed, 247 insertions(+), 99 deletions(-)
 create mode 100644 simulation_no_freelist_sync_test.go

diff --git a/db.go b/db.go
index f352ff14..dd391a23 100644
--- a/db.go
+++ b/db.go
@@ -61,6 +61,11 @@ type DB struct {
 	// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
 	NoSync bool
 
+	// When true, skips syncing freelist to disk. This improves the database
+	// write performance under normal operation, but requires a full database
+	// re-sync during recovery.
+	NoFreelistSync bool
+
 	// When true, skips the truncate call when growing the database.
 	// Setting this to true is only safe on non-ext3/ext4 systems.
 	// Skipping truncation avoids preallocation of hard drive space and
@@ -156,6 +161,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	}
 	db.NoGrowSync = options.NoGrowSync
 	db.MmapFlags = options.MmapFlags
+	db.NoFreelistSync = options.NoFreelistSync
 
 	// Set default values for later DB operations.
 	db.MaxBatchSize = DefaultMaxBatchSize
@@ -232,9 +238,14 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
-	// Read in the freelist.
-	db.freelist = newFreelist()
-	db.freelist.read(db.page(db.meta().freelist))
+	if db.NoFreelistSync {
+		db.freelist = newFreelist()
+		db.freelist.readIDs(db.freepages())
+	} else {
+		// Read in the freelist.
+		db.freelist = newFreelist()
+		db.freelist.read(db.page(db.meta().freelist))
+	}
 
 	// Mark the database as opened and return.
 	return db, nil
@@ -893,6 +904,38 @@ func (db *DB) IsReadOnly() bool {
 	return db.readOnly
 }
 
+func (db *DB) freepages() []pgid {
+	tx, err := db.beginTx()
+	defer func() {
+		err = tx.Rollback()
+		if err != nil {
+			panic("freepages: failed to rollback tx")
+		}
+	}()
+	if err != nil {
+		panic("freepages: failed to open read only tx")
+	}
+
+	reachable := make(map[pgid]*page)
+	nofreed := make(map[pgid]bool)
+	ech := make(chan error)
+	go func() {
+		for e := range ech {
+			panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e))
+		}
+	}()
+	tx.checkBucket(&tx.root, reachable, nofreed, ech)
+	close(ech)
+
+	var fids []pgid
+	for i := pgid(2); i < db.meta().pgid; i++ {
+		if _, ok := reachable[i]; !ok {
+			fids = append(fids, i)
+		}
+	}
+	return fids
+}
+
 // Options represents the options that can be set when opening a database.
 type Options struct {
 	// Timeout is the amount of time to wait to obtain a file lock.
@@ -903,6 +946,10 @@ type Options struct {
 	// Sets the DB.NoGrowSync flag before memory mapping the file.
 	NoGrowSync bool
 
+	// Do not sync freelist to disk. This improves the database write performance
+	// under normal operation, but requires a full database re-sync during recovery.
+	NoFreelistSync bool
+
 	// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
 	// grab a shared lock (UNIX).
 	ReadOnly bool
diff --git a/db_test.go b/db_test.go
index ad0e3d47..7a9afc27 100644
--- a/db_test.go
+++ b/db_test.go
@@ -1366,15 +1366,35 @@ func validateBatchBench(b *testing.B, db *DB) {
 // DB is a test wrapper for bolt.DB.
 type DB struct {
 	*bolt.DB
+	f string
+	o *bolt.Options
 }
 
 // MustOpenDB returns a new, open DB at a temporary location.
 func MustOpenDB() *DB {
-	db, err := bolt.Open(tempfile(), 0666, nil)
+	f := tempfile()
+	db, err := bolt.Open(f, 0666, nil)
 	if err != nil {
 		panic(err)
 	}
-	return &DB{db}
+	return &DB{
+		DB: db,
+		f:  f,
+	}
+}
+
+// MustOpenDBWithOption returns a new, open DB at a temporary location with given options.
+func MustOpenWithOption(o *bolt.Options) *DB {
+	f := tempfile()
+	db, err := bolt.Open(f, 0666, o)
+	if err != nil {
+		panic(err)
+	}
+	return &DB{
+		DB: db,
+		f:  f,
+		o:  o,
+	}
 }
 
 // Close closes the database and deletes the underlying file.
@@ -1399,6 +1419,15 @@ func (db *DB) MustClose() {
 	}
 }
 
+// MustReopen reopen the database. Panic on error.
+func (db *DB) MustReopen() {
+	indb, err := bolt.Open(db.f, 0666, db.o)
+	if err != nil {
+		panic(err)
+	}
+	db.DB = indb
+}
+
 // PrintStats prints the database stats
 func (db *DB) PrintStats() {
 	var stats = db.Stats()
diff --git a/freelist.go b/freelist.go
index aba48f58..f59dc074 100644
--- a/freelist.go
+++ b/freelist.go
@@ -185,6 +185,12 @@ func (f *freelist) read(p *page) {
 	f.reindex()
 }
 
+// read initializes the freelist from a given list of ids.
+func (f *freelist) readIDs(ids []pgid) {
+	f.ids = ids
+	f.reindex()
+}
+
 // write writes the page ids onto a freelist page. All free and pending ids are
 // saved to disk since in the event of a program crash, all pending ids will
 // become free.
diff --git a/simulation_no_freelist_sync_test.go b/simulation_no_freelist_sync_test.go
new file mode 100644
index 00000000..da2031ee
--- /dev/null
+++ b/simulation_no_freelist_sync_test.go
@@ -0,0 +1,47 @@
+package bolt_test
+
+import (
+	"testing"
+
+	"github.com/coreos/bbolt"
+)
+
+func TestSimulateNoFreeListSync_1op_1p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1, 1)
+}
+func TestSimulateNoFreeListSync_10op_1p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10, 1)
+}
+func TestSimulateNoFreeListSync_100op_1p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 1)
+}
+func TestSimulateNoFreeListSync_1000op_1p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 1)
+}
+func TestSimulateNoFreeListSync_10000op_1p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 1)
+}
+func TestSimulateNoFreeListSync_10op_10p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10, 10)
+}
+func TestSimulateNoFreeListSync_100op_10p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 10)
+}
+func TestSimulateNoFreeListSync_1000op_10p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 10)
+}
+func TestSimulateNoFreeListSync_10000op_10p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 10)
+}
+func TestSimulateNoFreeListSync_100op_100p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 100, 100)
+}
+func TestSimulateNoFreeListSync_1000op_100p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 1000, 100)
+}
+func TestSimulateNoFreeListSync_10000op_100p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 100)
+}
+func TestSimulateNoFreeListSync_10000op_1000p(t *testing.T) {
+	testSimulate(t, &bolt.Options{NoFreelistSync: true}, 8, 10000, 1000)
+}
diff --git a/simulation_test.go b/simulation_test.go
index e6241530..a5889c02 100644
--- a/simulation_test.go
+++ b/simulation_test.go
@@ -10,25 +10,25 @@ import (
 	"github.com/coreos/bbolt"
 )
 
-func TestSimulate_1op_1p(t *testing.T)     { testSimulate(t, 1, 1) }
-func TestSimulate_10op_1p(t *testing.T)    { testSimulate(t, 10, 1) }
-func TestSimulate_100op_1p(t *testing.T)   { testSimulate(t, 100, 1) }
-func TestSimulate_1000op_1p(t *testing.T)  { testSimulate(t, 1000, 1) }
-func TestSimulate_10000op_1p(t *testing.T) { testSimulate(t, 10000, 1) }
+func TestSimulate_1op_1p(t *testing.T)     { testSimulate(t, nil, 1, 1, 1) }
+func TestSimulate_10op_1p(t *testing.T)    { testSimulate(t, nil, 1, 10, 1) }
+func TestSimulate_100op_1p(t *testing.T)   { testSimulate(t, nil, 1, 100, 1) }
+func TestSimulate_1000op_1p(t *testing.T)  { testSimulate(t, nil, 1, 1000, 1) }
+func TestSimulate_10000op_1p(t *testing.T) { testSimulate(t, nil, 1, 10000, 1) }
 
-func TestSimulate_10op_10p(t *testing.T)    { testSimulate(t, 10, 10) }
-func TestSimulate_100op_10p(t *testing.T)   { testSimulate(t, 100, 10) }
-func TestSimulate_1000op_10p(t *testing.T)  { testSimulate(t, 1000, 10) }
-func TestSimulate_10000op_10p(t *testing.T) { testSimulate(t, 10000, 10) }
+func TestSimulate_10op_10p(t *testing.T)    { testSimulate(t, nil, 1, 10, 10) }
+func TestSimulate_100op_10p(t *testing.T)   { testSimulate(t, nil, 1, 100, 10) }
+func TestSimulate_1000op_10p(t *testing.T)  { testSimulate(t, nil, 1, 1000, 10) }
+func TestSimulate_10000op_10p(t *testing.T) { testSimulate(t, nil, 1, 10000, 10) }
 
-func TestSimulate_100op_100p(t *testing.T)   { testSimulate(t, 100, 100) }
-func TestSimulate_1000op_100p(t *testing.T)  { testSimulate(t, 1000, 100) }
-func TestSimulate_10000op_100p(t *testing.T) { testSimulate(t, 10000, 100) }
+func TestSimulate_100op_100p(t *testing.T)   { testSimulate(t, nil, 1, 100, 100) }
+func TestSimulate_1000op_100p(t *testing.T)  { testSimulate(t, nil, 1, 1000, 100) }
+func TestSimulate_10000op_100p(t *testing.T) { testSimulate(t, nil, 1, 10000, 100) }
 
-func TestSimulate_10000op_1000p(t *testing.T) { testSimulate(t, 10000, 1000) }
+func TestSimulate_10000op_1000p(t *testing.T) { testSimulate(t, nil, 1, 10000, 1000) }
 
 // Randomly generate operations on a given database with multiple clients to ensure consistency and thread safety.
-func testSimulate(t *testing.T, threadCount, parallelism int) {
+func testSimulate(t *testing.T, openOption *bolt.Options, round, threadCount, parallelism int) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}
@@ -42,81 +42,88 @@ func testSimulate(t *testing.T, threadCount, parallelism int) {
 	var versions = make(map[int]*QuickDB)
 	versions[1] = NewQuickDB()
 
-	db := MustOpenDB()
+	db := MustOpenWithOption(openOption)
 	defer db.MustClose()
 
 	var mutex sync.Mutex
 
 	// Run n threads in parallel, each with their own operation.
 	var wg sync.WaitGroup
-	var threads = make(chan bool, parallelism)
-	var i int
-	for {
-		threads <- true
-		wg.Add(1)
-		writable := ((rand.Int() % 100) < 20) // 20% writers
-
-		// Choose an operation to execute.
-		var handler simulateHandler
-		if writable {
-			handler = writerHandlers[rand.Intn(len(writerHandlers))]
-		} else {
-			handler = readerHandlers[rand.Intn(len(readerHandlers))]
-		}
-
-		// Execute a thread for the given operation.
-		go func(writable bool, handler simulateHandler) {
-			defer wg.Done()
 
-			// Start transaction.
-			tx, err := db.Begin(writable)
-			if err != nil {
-				t.Fatal("tx begin: ", err)
-			}
+	for n := 0; n < round; n++ {
 
-			// Obtain current state of the dataset.
-			mutex.Lock()
-			var qdb = versions[tx.ID()]
-			if writable {
-				qdb = versions[tx.ID()-1].Copy()
-			}
-			mutex.Unlock()
+		var threads = make(chan bool, parallelism)
+		var i int
+		for {
+			threads <- true
+			wg.Add(1)
+			writable := ((rand.Int() % 100) < 20) // 20% writers
 
-			// Make sure we commit/rollback the tx at the end and update the state.
+			// Choose an operation to execute.
+			var handler simulateHandler
 			if writable {
-				defer func() {
-					mutex.Lock()
-					versions[tx.ID()] = qdb
-					mutex.Unlock()
-
-					if err := tx.Commit(); err != nil {
-						t.Fatal(err)
-					}
-				}()
+				handler = writerHandlers[rand.Intn(len(writerHandlers))]
 			} else {
-				defer func() { _ = tx.Rollback() }()
+				handler = readerHandlers[rand.Intn(len(readerHandlers))]
 			}
 
-			// Ignore operation if we don't have data yet.
-			if qdb == nil {
-				return
+			// Execute a thread for the given operation.
+			go func(writable bool, handler simulateHandler) {
+				defer wg.Done()
+
+				// Start transaction.
+				tx, err := db.Begin(writable)
+				if err != nil {
+					t.Fatal("tx begin: ", err)
+				}
+
+				// Obtain current state of the dataset.
+				mutex.Lock()
+				var qdb = versions[tx.ID()]
+				if writable {
+					qdb = versions[tx.ID()-1].Copy()
+				}
+				mutex.Unlock()
+
+				// Make sure we commit/rollback the tx at the end and update the state.
+				if writable {
+					defer func() {
+						mutex.Lock()
+						versions[tx.ID()] = qdb
+						mutex.Unlock()
+
+						if err := tx.Commit(); err != nil {
+							t.Fatal(err)
+						}
+					}()
+				} else {
+					defer func() { _ = tx.Rollback() }()
+				}
+
+				// Ignore operation if we don't have data yet.
+				if qdb == nil {
+					return
+				}
+
+				// Execute handler.
+				handler(tx, qdb)
+
+				// Release a thread back to the scheduling loop.
+				<-threads
+			}(writable, handler)
+
+			i++
+			if i > threadCount {
+				break
 			}
+		}
 
-			// Execute handler.
-			handler(tx, qdb)
-
-			// Release a thread back to the scheduling loop.
-			<-threads
-		}(writable, handler)
+		// Wait until all threads are done.
+		wg.Wait()
 
-		i++
-		if i > threadCount {
-			break
-		}
+		db.MustClose()
+		db.MustReopen()
 	}
-
-	// Wait until all threads are done.
-	wg.Wait()
 }
 
 type simulateHandler func(tx *bolt.Tx, qdb *QuickDB)
diff --git a/tx.go b/tx.go
index 6700308a..97083142 100644
--- a/tx.go
+++ b/tx.go
@@ -169,26 +169,9 @@ func (tx *Tx) Commit() error {
 	// Free the old root bucket.
 	tx.meta.root.root = tx.root.root
 
-	opgid := tx.meta.pgid
-
-	// Free the freelist and allocate new pages for it. This will overestimate
-	// the size of the freelist but not underestimate the size (which would be bad).
-	tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
-	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
-	if err != nil {
-		tx.rollback()
-		return err
-	}
-	if err := tx.db.freelist.write(p); err != nil {
-		tx.rollback()
-		return err
-	}
-	tx.meta.freelist = p.id
-
-	// If the high water mark has moved up then attempt to grow the database.
-	if tx.meta.pgid > opgid {
-		if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
-			tx.rollback()
+	if !tx.db.NoFreelistSync {
+		err := tx.commitFreelist()
+		if err != nil {
 			return err
 		}
 	}
@@ -235,6 +218,33 @@ func (tx *Tx) Commit() error {
 	return nil
 }
 
+func (tx *Tx) commitFreelist() error {
+	opgid := tx.meta.pgid
+
+	// Free the freelist and allocate new pages for it. This will overestimate
+	// the size of the freelist but not underestimate the size (which would be bad).
+	tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
+	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
+	if err != nil {
+		tx.rollback()
+		return err
+	}
+	if err := tx.db.freelist.write(p); err != nil {
+		tx.rollback()
+		return err
+	}
+	tx.meta.freelist = p.id
+	// If the high water mark has moved up then attempt to grow the database.
+	if tx.meta.pgid > opgid {
+		if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
+			tx.rollback()
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Rollback closes the transaction and ignores all previous updates. Read-only
 // transactions must be rolled back and not committed.
 func (tx *Tx) Rollback() error {
@@ -394,8 +404,10 @@ func (tx *Tx) check(ch chan error) {
 	reachable := make(map[pgid]*page)
 	reachable[0] = tx.page(0) // meta0
 	reachable[1] = tx.page(1) // meta1
-	for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
-		reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
+	if !tx.DB().NoFreelistSync {
+		for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
+			reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
+		}
 	}
 
 	// Recursively check buckets.

From 05bfb3bd9a086466d15e1d84cd681c48c28fc9bc Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Mon, 24 Jul 2017 18:42:20 -0700
Subject: [PATCH 05/59] rebuild freelist when opening with FreelistSync after
 NoFreelistSync

Writes pgidNoFreelist to the meta freelist page to detect when freelists
haven't been synced down.

Fixes #5
---
 db.go      | 28 ++++++++++++++++++----
 db_test.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tx.go      |  6 ++++-
 3 files changed, 96 insertions(+), 6 deletions(-)

diff --git a/db.go b/db.go
index a6ab73c9..c9d800fc 100644
--- a/db.go
+++ b/db.go
@@ -24,6 +24,8 @@ const version = 2
 // Represents a marker value to indicate that a file is a Bolt DB.
 const magic uint32 = 0xED0CDAED
 
+const pgidNoFreelist pgid = 0xffffffffffffffff
+
 // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when
 // syncing changes to a file.  This is required as some operating systems,
 // such as OpenBSD, do not have a unified buffer cache (UBC) and writes
@@ -239,14 +241,29 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
-	if db.NoFreelistSync {
-		db.freelist = newFreelist()
+	db.freelist = newFreelist()
+	noFreeList := db.meta().freelist == pgidNoFreelist
+	if noFreeList {
+		// Reconstruct free list by scanning the DB.
 		db.freelist.readIDs(db.freepages())
 	} else {
-		// Read in the freelist.
-		db.freelist = newFreelist()
+		// Read free list from freelist page.
 		db.freelist.read(db.page(db.meta().freelist))
 	}
+	db.stats.FreePageN = len(db.freelist.ids)
+
+	// Flush freelist when transitioning from no sync to sync so
+	// NoFreelistSync unaware boltdb can open the db later.
+	if !db.NoFreelistSync && noFreeList && ((mode & 0222) != 0) {
+		tx, err := db.Begin(true)
+		if tx != nil {
+			err = tx.Commit()
+		}
+		if err != nil {
+			_ = db.close()
+			return nil, err
+		}
+	}
 
 	// Mark the database as opened and return.
 	return db, nil
@@ -1065,7 +1082,8 @@ func (m *meta) copy(dest *meta) {
 func (m *meta) write(p *page) {
 	if m.root.root >= m.pgid {
 		panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid))
-	} else if m.freelist >= m.pgid {
+	} else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist {
+		// TODO: reject pgidNoFreeList if !NoFreelistSync
 		panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid))
 	}
 
diff --git a/db_test.go b/db_test.go
index 7a9afc27..f667eaef 100644
--- a/db_test.go
+++ b/db_test.go
@@ -422,6 +422,74 @@ func TestDB_Open_InitialMmapSize(t *testing.T) {
 	}
 }
 
+// TestOpen_RecoverFreeList tests opening the DB with free-list
+// write-out after no free list sync will recover the free list
+// and write it out.
+func TestOpen_RecoverFreeList(t *testing.T) {
+	db := MustOpenWithOption(&bolt.Options{NoFreelistSync: true})
+	defer db.MustClose()
+
+	// Write some pages.
+	tx, err := db.Begin(true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wbuf := make([]byte, 8192)
+	for i := 0; i < 100; i++ {
+		s := fmt.Sprintf("%d", i)
+		b, err := tx.CreateBucket([]byte(s))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err = b.Put([]byte(s), wbuf); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate free pages.
+	if tx, err = db.Begin(true); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 50; i++ {
+		s := fmt.Sprintf("%d", i)
+		b := tx.Bucket([]byte(s))
+		if b == nil {
+			t.Fatal(err)
+		}
+		if err := b.Delete([]byte(s)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Record freelist count from opening with NoFreelistSync.
+	db.MustReopen()
+	freepages := db.Stats().FreePageN
+	if freepages == 0 {
+		t.Fatalf("no free pages on NoFreelistSync reopen")
+	}
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check free page count is reconstructed when opened with freelist sync.
+	db.o = &bolt.Options{}
+	db.MustReopen()
+	// One less free page for syncing the free list on open.
+	freepages--
+	if fp := db.Stats().FreePageN; fp < freepages {
+		t.Fatalf("closed with %d free pages, opened with %d", freepages, fp)
+	}
+}
+
 // Ensure that a database cannot open a transaction when it's not open.
 func TestDB_Begin_ErrDatabaseNotOpen(t *testing.T) {
 	var db bolt.DB
diff --git a/tx.go b/tx.go
index 5370e5fe..fa1fa509 100644
--- a/tx.go
+++ b/tx.go
@@ -174,6 +174,8 @@ func (tx *Tx) Commit() error {
 		if err != nil {
 			return err
 		}
+	} else {
+		tx.meta.freelist = pgidNoFreelist
 	}
 
 	// Write dirty pages to disk.
@@ -223,7 +225,9 @@ func (tx *Tx) commitFreelist() error {
 
 	// Free the freelist and allocate new pages for it. This will overestimate
 	// the size of the freelist but not underestimate the size (which would be bad).
-	tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
+	if tx.meta.freelist != pgidNoFreelist {
+		tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
+	}
 	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
 	if err != nil {
 		tx.rollback()

From 7ce671beeec4d67990e63bb41f08cbd16803e300 Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Thu, 13 Jul 2017 09:47:45 -0700
Subject: [PATCH 06/59] *: fix gofmt style issues in 'range'

---
 db_test.go  | 4 ++--
 freelist.go | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/db_test.go b/db_test.go
index f667eaef..46db4d21 100644
--- a/db_test.go
+++ b/db_test.go
@@ -1587,7 +1587,7 @@ func mustContainKeys(b *bolt.Bucket, m map[string]string) {
 
 	// Check for keys found in bucket that shouldn't be there.
 	var keys []string
-	for k, _ := range found {
+	for k := range found {
 		if _, ok := m[string(k)]; !ok {
 			keys = append(keys, k)
 		}
@@ -1598,7 +1598,7 @@ func mustContainKeys(b *bolt.Bucket, m map[string]string) {
 	}
 
 	// Check for keys not found in bucket that should be there.
-	for k, _ := range m {
+	for k := range m {
 		if _, ok := found[string(k)]; !ok {
 			keys = append(keys, k)
 		}
diff --git a/freelist.go b/freelist.go
index 8ac0d61d..679fcbb8 100644
--- a/freelist.go
+++ b/freelist.go
@@ -6,7 +6,6 @@ import (
 	"unsafe"
 )
 
-
 // txPending holds a list of pgids and corresponding allocation txns
 // that are pending to be freed.
 type txPending struct {

From a30394a020e13a912fc39a9a44636e7db580a6cf Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Thu, 27 Jul 2017 11:11:07 -0700
Subject: [PATCH 07/59] *: update git paths to 'coreos/bbolt'

---
 Makefile         | 4 ++--
 cmd/bolt/main.go | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index e035e63a..866184e1 100644
--- a/Makefile
+++ b/Makefile
@@ -9,9 +9,9 @@ race:
 
 # go get github.com/kisielk/errcheck
 errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/boltdb/bolt
+	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt
 
-test: 
+test:
 	@go test -v -cover .
 	@go test -v ./cmd/bolt
 
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 057eca50..387ce06e 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -19,7 +19,7 @@ import (
 	"unicode/utf8"
 	"unsafe"
 
-	"github.com/boltdb/bolt"
+	bolt "github.com/coreos/bbolt"
 )
 
 var (

From f50ad8e90c3cca5b8f6ee339a486e3fa0367cd44 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Tue, 8 Aug 2017 19:47:15 -0700
Subject: [PATCH 08/59] test: check free page counts on close/reopen for
 freelist overflow

Confirm that the number of freed pages exceeds the overflow count, then check
that reopening gives the same number of free pages.
---
 bucket_test.go | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/bucket_test.go b/bucket_test.go
index bcdd1dce..f044142a 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -423,6 +423,22 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
+
+	// Check more than an overflow's worth of pages are freed.
+	stats := db.Stats()
+	freePages := stats.FreePageN + stats.PendingPageN
+	if freePages <= 0xFFFF {
+		t.Fatalf("expected more than 0xFFFF free pages, got %v", freePages)
+	}
+
+	// Free page count should be preserved on reopen.
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db.MustReopen()
+	if reopenFreePages := db.Stats().FreePageN; freePages != reopenFreePages {
+		t.Fatalf("expected %d free pages, got %+v", freePages, db.Stats())
+	}
 }
 
 // Ensure that accessing and updating nested buckets is ok across transactions.

From 03f5e16968ca28cba6431ad6a863b9021bf97d19 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Tue, 8 Aug 2017 23:34:56 -0700
Subject: [PATCH 09/59] freelist: read all free pages on count overflow

count is not shifted up by start index when taking subslice of free
list, dropping the last entry in the list.
---
 freelist.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freelist.go b/freelist.go
index 679fcbb8..13ce5166 100644
--- a/freelist.go
+++ b/freelist.go
@@ -245,7 +245,7 @@ func (f *freelist) read(p *page) {
 	if count == 0 {
 		f.ids = nil
 	} else {
-		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count]
+		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:idx+count]
 		f.ids = make([]pgid, len(ids))
 		copy(f.ids, ids)
 

From 1038faf165e8b53109d1b95527b709f90596c6cb Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Tue, 8 Aug 2017 14:28:39 -0700
Subject: [PATCH 10/59] test: use qconfig() instead of defaults in
 TestBucket_Put_Single

Default/nil quick.Config uses 1000 rounds, causing TestBucker_Put_Single to
run for over 3 minutes in CI. The default count (via qconfig()) for boltdb
is 5, so use that.
---
 bucket_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bucket_test.go b/bucket_test.go
index bcdd1dce..bf1b48a8 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -1640,7 +1640,7 @@ func TestBucket_Put_Single(t *testing.T) {
 
 		index++
 		return true
-	}, nil); err != nil {
+	}, qconfig()); err != nil {
 		t.Error(err)
 	}
 }

From fd5de8495a5799aaaaac22c130a4ec32fcf459ef Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Mon, 7 Aug 2017 19:42:27 -0700
Subject: [PATCH 11/59] fix NoSyncFreelist reachability checking

* unconditionally free freelist, if any, when committing txn

* only treat freelist pages as reachable if set to valid pgid

Fixes #9
---
 tx.go | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tx.go b/tx.go
index fa1fa509..6b2fa283 100644
--- a/tx.go
+++ b/tx.go
@@ -169,6 +169,11 @@ func (tx *Tx) Commit() error {
 	// Free the old root bucket.
 	tx.meta.root.root = tx.root.root
 
+	// Free the old freelist because commit writes out a fresh freelist.
+	if tx.meta.freelist != pgidNoFreelist {
+		tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
+	}
+
 	if !tx.db.NoFreelistSync {
 		err := tx.commitFreelist()
 		if err != nil {
@@ -221,13 +226,9 @@ func (tx *Tx) Commit() error {
 }
 
 func (tx *Tx) commitFreelist() error {
-	opgid := tx.meta.pgid
-
-	// Free the freelist and allocate new pages for it. This will overestimate
+	// Allocate new pages for the new free list. This will overestimate
 	// the size of the freelist but not underestimate the size (which would be bad).
-	if tx.meta.freelist != pgidNoFreelist {
-		tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
-	}
+	opgid := tx.meta.pgid
 	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
 	if err != nil {
 		tx.rollback()
@@ -408,7 +409,7 @@ func (tx *Tx) check(ch chan error) {
 	reachable := make(map[pgid]*page)
 	reachable[0] = tx.page(0) // meta0
 	reachable[1] = tx.page(1) // meta1
-	if !tx.DB().NoFreelistSync {
+	if tx.meta.freelist != pgidNoFreelist {
 		for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
 			reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
 		}

From 78ca4fde003ef1d1aa3ef53c308e7d1225000606 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 16:43:35 -0700
Subject: [PATCH 12/59] get rid of os.Getpagesize() calls where appropriate

---
 bucket_test.go | 10 +++++-----
 db.go          |  4 ++--
 db_test.go     |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/bucket_test.go b/bucket_test.go
index c75f71ff..ead8ad74 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -1214,7 +1214,7 @@ func TestBucket_Stats(t *testing.T) {
 		}
 
 		// Only check allocations for 4KB pages.
-		if os.Getpagesize() == 4096 {
+		if db.Info().PageSize == 4096 {
 			if stats.BranchAlloc != 4096 {
 				t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc)
 			} else if stats.LeafAlloc != 36864 {
@@ -1347,7 +1347,7 @@ func TestBucket_Stats_Small(t *testing.T) {
 			t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse)
 		}
 
-		if os.Getpagesize() == 4096 {
+		if db.Info().PageSize == 4096 {
 			if stats.BranchAlloc != 0 {
 				t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc)
 			} else if stats.LeafAlloc != 0 {
@@ -1406,7 +1406,7 @@ func TestBucket_Stats_EmptyBucket(t *testing.T) {
 			t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse)
 		}
 
-		if os.Getpagesize() == 4096 {
+		if db.Info().PageSize == 4096 {
 			if stats.BranchAlloc != 0 {
 				t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc)
 			} else if stats.LeafAlloc != 0 {
@@ -1508,7 +1508,7 @@ func TestBucket_Stats_Nested(t *testing.T) {
 			t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse)
 		}
 
-		if os.Getpagesize() == 4096 {
+		if db.Info().PageSize == 4096 {
 			if stats.BranchAlloc != 0 {
 				t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc)
 			} else if stats.LeafAlloc != 8192 {
@@ -1581,7 +1581,7 @@ func TestBucket_Stats_Large(t *testing.T) {
 			t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse)
 		}
 
-		if os.Getpagesize() == 4096 {
+		if db.Info().PageSize == 4096 {
 			if stats.BranchAlloc != 53248 {
 				t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc)
 			} else if stats.LeafAlloc != 4898816 {
diff --git a/db.go b/db.go
index c9d800fc..7c952b5c 100644
--- a/db.go
+++ b/db.go
@@ -221,7 +221,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 				// If the first page is invalid and this OS uses a different
 				// page size than what the database was created with then we
 				// are out of luck and cannot access the database.
-				db.pageSize = os.Getpagesize()
+				db.pageSize = defaultPageSize
 			} else {
 				db.pageSize = int(m.pageSize)
 			}
@@ -371,7 +371,7 @@ func (db *DB) mmapSize(size int) (int, error) {
 // init creates a new database file and initializes its meta pages.
 func (db *DB) init() error {
 	// Set the page size to the OS page size.
-	db.pageSize = os.Getpagesize()
+	db.pageSize = defaultPageSize
 
 	// Create two meta pages on a buffer.
 	buf := make([]byte, db.pageSize*4)
diff --git a/db_test.go b/db_test.go
index 46db4d21..0b03bc9d 100644
--- a/db_test.go
+++ b/db_test.go
@@ -347,12 +347,13 @@ func TestOpen_FileTooSmall(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	pageSize := int64(db.Info().PageSize)
 	if err := db.Close(); err != nil {
 		t.Fatal(err)
 	}
 
 	// corrupt the database
-	if err := os.Truncate(path, int64(os.Getpagesize())); err != nil {
+	if err := os.Truncate(path, pageSize); err != nil {
 		t.Fatal(err)
 	}
 

From 012f88489b620a08d097a8972419f4f3d9f81d92 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 17:22:05 -0700
Subject: [PATCH 13/59] add PageSize to Option struct

Configure the db page size at runtime. Makes cross-arch debugging a bit easier.
---
 db.go      | 36 +++++++++++++++++++++---------------
 db_test.go | 16 ++++++++++++++++
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/db.go b/db.go
index 7c952b5c..71328452 100644
--- a/db.go
+++ b/db.go
@@ -200,6 +200,11 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// Default values for test hooks
 	db.ops.writeAt = db.file.WriteAt
 
+	if db.pageSize = options.PageSize; db.pageSize == 0 {
+		// Set the default page size to the OS page size.
+		db.pageSize = defaultPageSize
+	}
+
 	// Initialize the database if it doesn't exist.
 	if info, err := db.file.Stat(); err != nil {
 		return nil, err
@@ -211,20 +216,21 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	} else {
 		// Read the first meta page to determine the page size.
 		var buf [0x1000]byte
-		if _, err := db.file.ReadAt(buf[:], 0); err == nil {
-			m := db.pageInBuffer(buf[:], 0).meta()
-			if err := m.validate(); err != nil {
-				// If we can't read the page size, we can assume it's the same
-				// as the OS -- since that's how the page size was chosen in the
-				// first place.
-				//
-				// If the first page is invalid and this OS uses a different
-				// page size than what the database was created with then we
-				// are out of luck and cannot access the database.
-				db.pageSize = defaultPageSize
-			} else {
+		// If we can't read the page size, but can read a page, assume
+		// it's the same as the OS or one given -- since that's how the
+		// page size was chosen in the first place.
+		//
+		// If the first page is invalid and this OS uses a different
+		// page size than what the database was created with then we
+		// are out of luck and cannot access the database.
+		//
+		// TODO: scan for next page
+		if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
+			if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
 				db.pageSize = int(m.pageSize)
 			}
+		} else {
+			return nil, ErrInvalid
 		}
 	}
 
@@ -370,9 +376,6 @@ func (db *DB) mmapSize(size int) (int, error) {
 
 // init creates a new database file and initializes its meta pages.
 func (db *DB) init() error {
-	// Set the page size to the OS page size.
-	db.pageSize = defaultPageSize
-
 	// Create two meta pages on a buffer.
 	buf := make([]byte, db.pageSize*4)
 	for i := 0; i < 2; i++ {
@@ -999,6 +1002,9 @@ type Options struct {
 	// If initialMmapSize is smaller than the previous database size,
 	// it takes no effect.
 	InitialMmapSize int
+
+	// PageSize overrides the default OS page size.
+	PageSize int
 }
 
 // DefaultOptions represent the options used if nil options are passed into Open().
diff --git a/db_test.go b/db_test.go
index 0b03bc9d..ebad50a4 100644
--- a/db_test.go
+++ b/db_test.go
@@ -423,6 +423,22 @@ func TestDB_Open_InitialMmapSize(t *testing.T) {
 	}
 }
 
+// TestOpen_BigPage checks the database uses bigger pages when
+// changing PageSize.
+func TestOpen_BigPage(t *testing.T) {
+	pageSize := os.Getpagesize()
+
+	db1 := MustOpenWithOption(&bolt.Options{PageSize: pageSize * 2})
+	defer db1.MustClose()
+
+	db2 := MustOpenWithOption(&bolt.Options{PageSize: pageSize * 4})
+	defer db2.MustClose()
+
+	if db1sz, db2sz := fileSize(db1.f), fileSize(db2.f); db1sz >= db2sz {
+		t.Errorf("expected %d < %d", db1sz, db2sz)
+	}
+}
+
 // TestOpen_RecoverFreeList tests opening the DB with free-list
 // write-out after no free list sync will recover the free list
 // and write it out.

From 319a33c534f3be1966b0cfcfa5eefa415249037d Mon Sep 17 00:00:00 2001
From: Christian Mauduit <christian.mauduit@scmfrance.fr>
Date: Wed, 13 Apr 2016 12:50:37 +0200
Subject: [PATCH 14/59] Fix issue #543 'bolt bench crashes'

Added a test to check that bucket is actually non-nil before getting a
cursor on it. This happens when querying non-bucket entries, the
Bucket() method does not return explicit errors, so we can guess it's
probably because it's a key of the wrong type (ie: not leaf). No clear
idea wether the results in the case described '-read-mode=seq
-write-mode=rnd-nest' really do make sense, it looks like read are
zero whatsoever.
---
 cmd/bolt/main.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 387ce06e..463a1c46 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -1183,12 +1183,14 @@ func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOpt
 			var count int
 			var top = tx.Bucket(benchBucketName)
 			if err := top.ForEach(func(name, _ []byte) error {
-				c := top.Bucket(name).Cursor()
-				for k, v := c.First(); k != nil; k, v = c.Next() {
-					if v == nil {
-						return ErrInvalidValue
+				if b := top.Bucket(name); b != nil {
+					c := b.Cursor()
+					for k, v := c.First(); k != nil; k, v = c.Next() {
+						if v == nil {
+							return ErrInvalidValue
+						}
+						count++
 					}
-					count++
 				}
 				return nil
 			}); err != nil {

From 89b9a2c19e2a1e178c1a637165ffd4e219246333 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 21:39:41 -0700
Subject: [PATCH 15/59] make fmt

* gosimple
* unused
* gofmt
---
 Makefile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Makefile b/Makefile
index 866184e1..937a892d 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,14 @@ default: build
 race:
 	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
 
+# go get honnef.co/go/tools/simple
+# go get honnef.co/go/tools/unused
+fmt:
+	gosimple ./...
+	unused ./...
+	gofmt -l -s -d $(find -name \*.go)
+
+
 # go get github.com/kisielk/errcheck
 errcheck:
 	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt

From e5514a24a6df91d2331212875763893fb8001cec Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 21:39:16 -0700
Subject: [PATCH 16/59] pass gosimple

---
 cmd/bolt/main.go | 14 +++-----------
 tx.go            |  5 +----
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 387ce06e..682ba081 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -188,17 +188,9 @@ func (cmd *CheckCommand) Run(args ...string) error {
 	// Perform consistency check.
 	return db.View(func(tx *bolt.Tx) error {
 		var count int
-		ch := tx.Check()
-	loop:
-		for {
-			select {
-			case err, ok := <-ch:
-				if !ok {
-					break loop
-				}
-				fmt.Fprintln(cmd.Stdout, err)
-				count++
-			}
+		for err := range tx.Check() {
+			fmt.Fprintln(cmd.Stdout, err)
+			count++
 		}
 
 		// Print summary of errors.
diff --git a/tx.go b/tx.go
index 6b2fa283..57c4d5ad 100644
--- a/tx.go
+++ b/tx.go
@@ -126,10 +126,7 @@ func (tx *Tx) DeleteBucket(name []byte) error {
 // the error is returned to the caller.
 func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error {
 	return tx.root.ForEach(func(k, v []byte) error {
-		if err := fn(k, tx.root.Bucket(k)); err != nil {
-			return err
-		}
-		return nil
+		return fn(k, tx.root.Bucket(k))
 	})
 }
 

From 32c9f9e92948460d6bf21933e0bacb61a1e02cf3 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 22:06:10 -0700
Subject: [PATCH 17/59] pass unused

---
 bucket.go        |  7 -------
 cmd/bolt/main.go | 10 ++--------
 db.go            | 15 +--------------
 db_test.go       | 48 ------------------------------------------------
 4 files changed, 3 insertions(+), 77 deletions(-)

diff --git a/bucket.go b/bucket.go
index 0c5bf274..176fa998 100644
--- a/bucket.go
+++ b/bucket.go
@@ -14,13 +14,6 @@ const (
 	MaxValueSize = (1 << 31) - 2
 )
 
-const (
-	maxUint = ^uint(0)
-	minUint = 0
-	maxInt  = int(^uint(0) >> 1)
-	minInt  = -maxInt - 1
-)
-
 const bucketHeaderSize = int(unsafe.Sizeof(bucket{}))
 
 const (
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 682ba081..e9a7770d 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -48,12 +48,6 @@ var (
 
 	// ErrPageIDRequired is returned when a required page id is not specified.
 	ErrPageIDRequired = errors.New("page id required")
-
-	// ErrPageNotFound is returned when specifying a page above the high water mark.
-	ErrPageNotFound = errors.New("page not found")
-
-	// ErrPageFreed is returned when reading a page that has already been freed.
-	ErrPageFreed = errors.New("page freed")
 )
 
 // PageHeaderSize represents the size of the bolt.page header.
@@ -1023,12 +1017,12 @@ func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, res
 
 func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
 	var i = uint32(0)
-	return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
+	return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { i++; return i })
 }
 
 func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
+	return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
 }
 
 func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
diff --git a/db.go b/db.go
index c9d800fc..db3843c7 100644
--- a/db.go
+++ b/db.go
@@ -7,9 +7,7 @@ import (
 	"log"
 	"os"
 	"runtime"
-	"runtime/debug"
 	"sort"
-	"strings"
 	"sync"
 	"time"
 	"unsafe"
@@ -193,6 +191,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// The database file is locked using the shared lock (more than one process may
 	// hold a lock at the same time) otherwise (options.ReadOnly is set).
 	if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil {
+		db.lockfile = nil // make 'unused' happy. TODO: rework locks
 		_ = db.close()
 		return nil, err
 	}
@@ -1040,10 +1039,6 @@ func (s *Stats) Sub(other *Stats) Stats {
 	return diff
 }
 
-func (s *Stats) add(other *Stats) {
-	s.TxStats.add(&other.TxStats)
-}
-
 type Info struct {
 	Data     uintptr
 	PageSize int
@@ -1110,11 +1105,3 @@ func _assert(condition bool, msg string, v ...interface{}) {
 		panic(fmt.Sprintf("assertion failed: "+msg, v...))
 	}
 }
-
-func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
-func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
-
-func printstack() {
-	stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n")
-	fmt.Fprintln(os.Stderr, stack)
-}
diff --git a/db_test.go b/db_test.go
index 46db4d21..dd09a4db 100644
--- a/db_test.go
+++ b/db_test.go
@@ -12,8 +12,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"sort"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -24,12 +22,6 @@ import (
 
 var statsFlag = flag.Bool("stats", false, "show performance stats")
 
-// version is the data file format version.
-const version = 2
-
-// magic is the marker value to indicate that a file is a Bolt DB.
-const magic uint32 = 0xED0CDAED
-
 // pageSize is the size of one page in the data file.
 const pageSize = 4096
 
@@ -1575,40 +1567,6 @@ func tempfile() string {
 	return f.Name()
 }
 
-// mustContainKeys checks that a bucket contains a given set of keys.
-func mustContainKeys(b *bolt.Bucket, m map[string]string) {
-	found := make(map[string]string)
-	if err := b.ForEach(func(k, _ []byte) error {
-		found[string(k)] = ""
-		return nil
-	}); err != nil {
-		panic(err)
-	}
-
-	// Check for keys found in bucket that shouldn't be there.
-	var keys []string
-	for k := range found {
-		if _, ok := m[string(k)]; !ok {
-			keys = append(keys, k)
-		}
-	}
-	if len(keys) > 0 {
-		sort.Strings(keys)
-		panic(fmt.Sprintf("keys found(%d): %s", len(keys), strings.Join(keys, ",")))
-	}
-
-	// Check for keys not found in bucket that should be there.
-	for k := range m {
-		if _, ok := found[string(k)]; !ok {
-			keys = append(keys, k)
-		}
-	}
-	if len(keys) > 0 {
-		sort.Strings(keys)
-		panic(fmt.Sprintf("keys not found(%d): %s", len(keys), strings.Join(keys, ",")))
-	}
-}
-
 func trunc(b []byte, length int) []byte {
 	if length < len(b) {
 		return b[:length]
@@ -1628,15 +1586,9 @@ func fileSize(path string) int64 {
 	return fi.Size()
 }
 
-func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
-func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
-
 // u64tob converts a uint64 into an 8-byte slice.
 func u64tob(v uint64) []byte {
 	b := make([]byte, 8)
 	binary.BigEndian.PutUint64(b, v)
 	return b
 }
-
-// btou64 converts an 8-byte slice into an uint64.
-func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }

From d3d8bbd794f772b2a62111fedce539e8a02db559 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 22:07:12 -0700
Subject: [PATCH 18/59] pass gofmt

---
 freelist.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freelist.go b/freelist.go
index 13ce5166..78e71cbf 100644
--- a/freelist.go
+++ b/freelist.go
@@ -245,7 +245,7 @@ func (f *freelist) read(p *page) {
 	if count == 0 {
 		f.ids = nil
 	} else {
-		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:idx+count]
+		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count]
 		f.ids = make([]pgid, len(ids))
 		copy(f.ids, ids)
 

From ec37ce8e9bebc0798d665847d8ff4ba05dc647f3 Mon Sep 17 00:00:00 2001
From: Sokolov Yura aka funny_falcon <funny.falcon@gmail.com>
Date: Sat, 15 Aug 2015 13:13:23 +0300
Subject: [PATCH 19/59] do not read freelist if database opened readonly

---
 db.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/db.go b/db.go
index c9d800fc..a180f3ba 100644
--- a/db.go
+++ b/db.go
@@ -241,6 +241,10 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
+	if db.readOnly {
+		return db, nil
+	}
+
 	db.freelist = newFreelist()
 	noFreeList := db.meta().freelist == pgidNoFreelist
 	if noFreeList {
@@ -254,7 +258,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 
 	// Flush freelist when transitioning from no sync to sync so
 	// NoFreelistSync unaware boltdb can open the db later.
-	if !db.NoFreelistSync && noFreeList && ((mode & 0222) != 0) {
+	if !db.NoFreelistSync && noFreeList {
 		tx, err := db.Begin(true)
 		if tx != nil {
 			err = tx.Commit()

From e264e743a906b327007dc669706b2fbf399f4ba6 Mon Sep 17 00:00:00 2001
From: Vladimir Stefanovic <vladimir.stefanovic@imgtec.com>
Date: Fri, 17 Feb 2017 20:44:43 +0100
Subject: [PATCH 20/59] Add support for mips, mips64

---
 bolt_mips64x.go | 12 ++++++++++++
 bolt_mipsx.go   | 12 ++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 bolt_mips64x.go
 create mode 100644 bolt_mipsx.go

diff --git a/bolt_mips64x.go b/bolt_mips64x.go
new file mode 100644
index 00000000..134b578b
--- /dev/null
+++ b/bolt_mips64x.go
@@ -0,0 +1,12 @@
+// +build mips64 mips64le
+
+package bolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x8000000000 // 512GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false
diff --git a/bolt_mipsx.go b/bolt_mipsx.go
new file mode 100644
index 00000000..d5ecb059
--- /dev/null
+++ b/bolt_mipsx.go
@@ -0,0 +1,12 @@
+// +build mips mipsle
+
+package bolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x40000000 // 1GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

From ef8e711cfb03569f16f4fd667d0c551526bf0459 Mon Sep 17 00:00:00 2001
From: dogben <dogben@users.noreply.github.com>
Date: Sun, 11 Jun 2017 18:52:05 -0400
Subject: [PATCH 21/59] Set FillPercent=1.0 in 'bolt compact'.

By default, pages are split when they reach half full. For 'bolt compact' we want to fill the entire page for maximum compaction.
---
 cmd/bolt/main.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 387ce06e..a007370d 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -1664,6 +1664,9 @@ func (cmd *CompactCommand) compact(dst, src *bolt.DB) error {
 			}
 		}
 
+		// Fill the entire page for best compaction.
+		b.FillPercent = 1.0
+
 		// If there is no value then this is a bucket call.
 		if v == nil {
 			bkt, err := b.CreateBucket(k)

From 3c6c3ac0a2ceb97115487baf586b3692550384f0 Mon Sep 17 00:00:00 2001
From: Pavel Borzenkov <pavel.borzenkov@gmail.com>
Date: Wed, 8 Feb 2017 18:24:56 +0300
Subject: [PATCH 22/59] Add a test for deletion of non-existing key

As of now the test fails and tries to delete the next key returned by
the cursor, which happens to be a nested bucket. That's why Delete is
fails.

Signed-off-by: Pavel Borzenkov <pavel.borzenkov@gmail.com>
---
 bucket_test.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/bucket_test.go b/bucket_test.go
index c75f71ff..6d96e662 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -441,6 +441,39 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
 	}
 }
 
+// Ensure that deleting of non-existing key is a no-op.
+func TestBucket_Delete_NonExisting(t *testing.T) {
+	db := MustOpenDB()
+	defer db.MustClose()
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucket([]byte("widgets"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err = b.CreateBucket([]byte("nested")); err != nil {
+			t.Fatal(err)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("widgets"))
+		if err := b.Delete([]byte("foo")); err != nil {
+			t.Fatal(err)
+		}
+		if b.Bucket([]byte("nested")) == nil {
+			t.Fatal("nested bucket has been deleted")
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // Ensure that accessing and updating nested buckets is ok across transactions.
 func TestBucket_Nested(t *testing.T) {
 	db := MustOpenDB()

From 6336a429d168668797fbcb4d1f9e3895ae8aa8e2 Mon Sep 17 00:00:00 2001
From: Pavel Borzenkov <pavel.borzenkov@gmail.com>
Date: Wed, 8 Feb 2017 18:25:52 +0300
Subject: [PATCH 23/59] Fix deletion of non-existing keys

Doc for Bucket.Delete says that a Delete() on non-existing key is a
no-op. Right now it tries to delete the next key returned by the
cursor. Fix this by checking for key equivalence before deletion.

 Fixes #349

Signed-off-by: Pavel Borzenkov <pavel.borzenkov@gmail.com>
---
 bucket.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bucket.go b/bucket.go
index 0c5bf274..93f9d951 100644
--- a/bucket.go
+++ b/bucket.go
@@ -323,7 +323,12 @@ func (b *Bucket) Delete(key []byte) error {
 
 	// Move cursor to correct position.
 	c := b.Cursor()
-	_, _, flags := c.seek(key)
+	k, _, flags := c.seek(key)
+
+	// Return nil if the key doesn't exist.
+	if !bytes.Equal(key, k) {
+		return nil
+	}
 
 	// Return an error if there is already existing bucket value.
 	if (flags & bucketLeafFlag) != 0 {

From 1ab9756653ac7e47e6a51212e189b76d930ec5db Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Thu, 10 Aug 2017 23:35:06 -0700
Subject: [PATCH 24/59] fix overflow breaking 32-bit test builds

---
 db_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_test.go b/db_test.go
index 46db4d21..7a449705 100644
--- a/db_test.go
+++ b/db_test.go
@@ -370,7 +370,7 @@ func TestDB_Open_InitialMmapSize(t *testing.T) {
 	path := tempfile()
 	defer os.Remove(path)
 
-	initMmapSize := 1 << 31  // 2GB
+	initMmapSize := 1 << 30  // 1GB
 	testWriteSize := 1 << 27 // 134MB
 
 	db, err := bolt.Open(path, 0666, &bolt.Options{InitialMmapSize: initMmapSize})

From f07641f90ac349e89b74073bb5193a7a45acbcf9 Mon Sep 17 00:00:00 2001
From: Lewis Marshall <lewis@lmars.net>
Date: Tue, 28 Jul 2015 11:42:13 +0100
Subject: [PATCH 25/59] Add "dump" and "page" commands to CLI usage

Signed-off-by: Lewis Marshall <lewis@lmars.net>
---
 cmd/bolt/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index e62657b2..adab14b8 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -127,8 +127,10 @@ The commands are:
     bench       run synthetic benchmark against bolt
     check       verifies integrity of bolt database
     compact     copies a bolt database, compacting it in the process
+    dump        print a hexidecimal dump of a single page
     info        print basic info
     help        print this screen
+    page        print one or more pages in human readable format
     pages       print list of pages with their types
     stats       iterate over all pages and generate usage stats
 

From 0ca39ebe25bfc8ac3b8312f2bd67378c1917ac28 Mon Sep 17 00:00:00 2001
From: Lewis Marshall <lewis@lmars.net>
Date: Tue, 28 Jul 2015 11:54:15 +0100
Subject: [PATCH 26/59] Add "buckets", "keys" and "get" CLI commands

These were previously removed but I find them quite useful so have
reintroduced them.

Signed-off-by: Lewis Marshall <lewis@lmars.net>
---
 cmd/bolt/main.go      | 227 ++++++++++++++++++++++++++++++++++++++++++
 cmd/bolt/main_test.go | 100 +++++++++++++++++++
 2 files changed, 327 insertions(+)

diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index adab14b8..7cd36ffc 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -48,6 +48,18 @@ var (
 
 	// ErrPageIDRequired is returned when a required page id is not specified.
 	ErrPageIDRequired = errors.New("page id required")
+
+	// ErrBucketRequired is returned when a bucket is not specified.
+	ErrBucketRequired = errors.New("bucket required")
+
+	// ErrBucketNotFound is returned when a bucket is not found.
+	ErrBucketNotFound = errors.New("bucket not found")
+
+	// ErrKeyRequired is returned when a key is not specified.
+	ErrKeyRequired = errors.New("key required")
+
+	// ErrKeyNotFound is returned when a key is not found.
+	ErrKeyNotFound = errors.New("key not found")
 )
 
 // PageHeaderSize represents the size of the bolt.page header.
@@ -94,14 +106,20 @@ func (m *Main) Run(args ...string) error {
 		return ErrUsage
 	case "bench":
 		return newBenchCommand(m).Run(args[1:]...)
+	case "buckets":
+		return newBucketsCommand(m).Run(args[1:]...)
 	case "check":
 		return newCheckCommand(m).Run(args[1:]...)
 	case "compact":
 		return newCompactCommand(m).Run(args[1:]...)
 	case "dump":
 		return newDumpCommand(m).Run(args[1:]...)
+	case "get":
+		return newGetCommand(m).Run(args[1:]...)
 	case "info":
 		return newInfoCommand(m).Run(args[1:]...)
+	case "keys":
+		return newKeysCommand(m).Run(args[1:]...)
 	case "page":
 		return newPageCommand(m).Run(args[1:]...)
 	case "pages":
@@ -125,10 +143,13 @@ Usage:
 The commands are:
 
     bench       run synthetic benchmark against bolt
+    buckets     print a list of buckets
     check       verifies integrity of bolt database
     compact     copies a bolt database, compacting it in the process
     dump        print a hexidecimal dump of a single page
+    get         print the value of a key in a bucket
     info        print basic info
+    keys        print a list of keys in a bucket
     help        print this screen
     page        print one or more pages in human readable format
     pages       print list of pages with their types
@@ -867,6 +888,212 @@ experience corruption, please submit a ticket to the Bolt project page:
 `, "\n")
 }
 
+// BucketsCommand represents the "buckets" command execution.
+type BucketsCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewBucketsCommand returns a BucketsCommand.
+func newBucketsCommand(m *Main) *BucketsCommand {
+	return &BucketsCommand{
+		Stdin:  m.Stdin,
+		Stdout: m.Stdout,
+		Stderr: m.Stderr,
+	}
+}
+
+// Run executes the command.
+func (cmd *BucketsCommand) Run(args ...string) error {
+	// Parse flags.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	help := fs.Bool("h", false, "")
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if *help {
+		fmt.Fprintln(cmd.Stderr, cmd.Usage())
+		return ErrUsage
+	}
+
+	// Require database path.
+	path := fs.Arg(0)
+	if path == "" {
+		return ErrPathRequired
+	} else if _, err := os.Stat(path); os.IsNotExist(err) {
+		return ErrFileNotFound
+	}
+
+	// Open database.
+	db, err := bolt.Open(path, 0666, nil)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Print buckets.
+	return db.View(func(tx *bolt.Tx) error {
+		return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
+			fmt.Fprintln(cmd.Stdout, string(name))
+			return nil
+		})
+	})
+}
+
+// Usage returns the help message.
+func (cmd *BucketsCommand) Usage() string {
+	return strings.TrimLeft(`
+usage: bolt buckets PATH
+
+Print a list of buckets.
+`, "\n")
+}
+
+// KeysCommand represents the "keys" command execution.
+type KeysCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewKeysCommand returns a KeysCommand.
+func newKeysCommand(m *Main) *KeysCommand {
+	return &KeysCommand{
+		Stdin:  m.Stdin,
+		Stdout: m.Stdout,
+		Stderr: m.Stderr,
+	}
+}
+
+// Run executes the command.
+func (cmd *KeysCommand) Run(args ...string) error {
+	// Parse flags.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	help := fs.Bool("h", false, "")
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if *help {
+		fmt.Fprintln(cmd.Stderr, cmd.Usage())
+		return ErrUsage
+	}
+
+	// Require database path and bucket.
+	path, bucket := fs.Arg(0), fs.Arg(1)
+	if path == "" {
+		return ErrPathRequired
+	} else if _, err := os.Stat(path); os.IsNotExist(err) {
+		return ErrFileNotFound
+	} else if bucket == "" {
+		return ErrBucketRequired
+	}
+
+	// Open database.
+	db, err := bolt.Open(path, 0666, nil)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Print keys.
+	return db.View(func(tx *bolt.Tx) error {
+		// Find bucket.
+		b := tx.Bucket([]byte(bucket))
+		if b == nil {
+			return ErrBucketNotFound
+		}
+
+		// Iterate over each key.
+		return b.ForEach(func(key, _ []byte) error {
+			fmt.Fprintln(cmd.Stdout, string(key))
+			return nil
+		})
+	})
+}
+
+// Usage returns the help message.
+func (cmd *KeysCommand) Usage() string {
+	return strings.TrimLeft(`
+usage: bolt keys PATH BUCKET
+
+Print a list of keys in the given bucket.
+`, "\n")
+}
+
+// GetCommand represents the "get" command execution.
+type GetCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewGetCommand returns a GetCommand.
+func newGetCommand(m *Main) *GetCommand {
+	return &GetCommand{
+		Stdin:  m.Stdin,
+		Stdout: m.Stdout,
+		Stderr: m.Stderr,
+	}
+}
+
+// Run executes the command.
+func (cmd *GetCommand) Run(args ...string) error {
+	// Parse flags.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	help := fs.Bool("h", false, "")
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if *help {
+		fmt.Fprintln(cmd.Stderr, cmd.Usage())
+		return ErrUsage
+	}
+
+	// Require database path, bucket and key.
+	path, bucket, key := fs.Arg(0), fs.Arg(1), fs.Arg(2)
+	if path == "" {
+		return ErrPathRequired
+	} else if _, err := os.Stat(path); os.IsNotExist(err) {
+		return ErrFileNotFound
+	} else if bucket == "" {
+		return ErrBucketRequired
+	} else if key == "" {
+		return ErrKeyRequired
+	}
+
+	// Open database.
+	db, err := bolt.Open(path, 0666, nil)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Print value.
+	return db.View(func(tx *bolt.Tx) error {
+		// Find bucket.
+		b := tx.Bucket([]byte(bucket))
+		if b == nil {
+			return ErrBucketNotFound
+		}
+
+		// Find value for given key.
+		val := b.Get([]byte(key))
+		if val == nil {
+			return ErrKeyNotFound
+		}
+
+		fmt.Fprintln(cmd.Stdout, string(val))
+		return nil
+	})
+}
+
+// Usage returns the help message.
+func (cmd *GetCommand) Usage() string {
+	return strings.TrimLeft(`
+usage: bolt get PATH BUCKET KEY
+
+Print the value of the given key in the given bucket.
+`, "\n")
+}
+
 var benchBucketName = []byte("bench")
 
 // BenchCommand represents the "bench" command execution.
diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go
index e3a834d2..16bf804a 100644
--- a/cmd/bolt/main_test.go
+++ b/cmd/bolt/main_test.go
@@ -146,6 +146,106 @@ func TestStatsCommand_Run(t *testing.T) {
 	}
 }
 
+// Ensure the "buckets" command can print a list of buckets.
+func TestBucketsCommand_Run(t *testing.T) {
+	db := MustOpen(0666, nil)
+	defer db.Close()
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		for _, name := range []string{"foo", "bar", "baz"} {
+			_, err := tx.CreateBucket([]byte(name))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	db.DB.Close()
+
+	expected := "bar\nbaz\nfoo\n"
+
+	// Run the command.
+	m := NewMain()
+	if err := m.Run("buckets", db.Path); err != nil {
+		t.Fatal(err)
+	} else if actual := m.Stdout.String(); actual != expected {
+		t.Fatalf("unexpected stdout:\n\n%s", actual)
+	}
+}
+
+// Ensure the "keys" command can print a list of keys for a bucket.
+func TestKeysCommand_Run(t *testing.T) {
+	db := MustOpen(0666, nil)
+	defer db.Close()
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		for _, name := range []string{"foo", "bar"} {
+			b, err := tx.CreateBucket([]byte(name))
+			if err != nil {
+				return err
+			}
+			for i := 0; i < 3; i++ {
+				key := fmt.Sprintf("%s-%d", name, i)
+				if err := b.Put([]byte(key), []byte{0}); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	db.DB.Close()
+
+	expected := "foo-0\nfoo-1\nfoo-2\n"
+
+	// Run the command.
+	m := NewMain()
+	if err := m.Run("keys", db.Path, "foo"); err != nil {
+		t.Fatal(err)
+	} else if actual := m.Stdout.String(); actual != expected {
+		t.Fatalf("unexpected stdout:\n\n%s", actual)
+	}
+}
+
+// Ensure the "get" command can print the value of a key in a bucket.
+func TestGetCommand_Run(t *testing.T) {
+	db := MustOpen(0666, nil)
+	defer db.Close()
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		for _, name := range []string{"foo", "bar"} {
+			b, err := tx.CreateBucket([]byte(name))
+			if err != nil {
+				return err
+			}
+			for i := 0; i < 3; i++ {
+				key := fmt.Sprintf("%s-%d", name, i)
+				val := fmt.Sprintf("val-%s-%d", name, i)
+				if err := b.Put([]byte(key), []byte(val)); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	db.DB.Close()
+
+	expected := "val-foo-1\n"
+
+	// Run the command.
+	m := NewMain()
+	if err := m.Run("get", db.Path, "foo", "foo-1"); err != nil {
+		t.Fatal(err)
+	} else if actual := m.Stdout.String(); actual != expected {
+		t.Fatalf("unexpected stdout:\n\n%s", actual)
+	}
+}
+
 // Main represents a test wrapper for main.Main that records output.
 type Main struct {
 	*main.Main

From 4d8f7f7f8d1093be63d1ef9377962f66f80a0841 Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Fri, 11 Aug 2017 12:58:44 -0700
Subject: [PATCH 27/59] README: update links to 'coreos/bbolt'

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d7f80e97..cce56866 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
-Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg)
+Bolt
 ====
 
+[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
+
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,
 fast, and reliable database for projects that don't require a full database
@@ -59,7 +61,7 @@ Shopify and Heroku use Bolt-backed services every day.
 To start using Bolt, install Go and run `go get`:
 
 ```sh
-$ go get github.com/boltdb/bolt/...
+$ go get github.com/coreos/bbolt/...
 ```
 
 This will retrieve the library and install the `bolt` command line utility into
@@ -79,7 +81,7 @@ package main
 import (
 	"log"
 
-	"github.com/boltdb/bolt"
+	bolt "github.com/coreos/bbolt"
 )
 
 func main() {
@@ -522,7 +524,7 @@ this from a read-only transaction, it will perform a hot backup and not block
 your other database reads and writes.
 
 By default, it will use a regular file handle which will utilize the operating
-system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx)
+system's page cache. See the [`Tx`](https://godoc.org/github.com/coreos/bbolt#Tx)
 documentation for information about optimizing for larger-than-RAM datasets.
 
 One common use case is to backup over HTTP so you can use tools like `cURL` to

From 9d07787d9f24e6ee51e5eec094afe74441e8f9eb Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Fri, 11 Aug 2017 13:41:18 -0700
Subject: [PATCH 28/59] README: add goreportcard badge

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cce56866..ba31d467 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 Bolt
 ====
 
-[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
+[![Go Report Card](https://goreportcard.com/badge/github.com/coreos/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/coreos/bbolt) [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,

From c3092301afc605f17c6cb21cdee6f14e14c7b1b1 Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Fri, 11 Aug 2017 13:47:10 -0700
Subject: [PATCH 29/59] *: go vet fixes

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 bucket_test.go   | 1 +
 cmd/bolt/main.go | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bucket_test.go b/bucket_test.go
index a06a3e62..b7ce32c1 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -113,6 +113,7 @@ func TestBucket_Get_Capacity(t *testing.T) {
 		// Ensure slice can be appended to without a segfault.
 		k = append(k, []byte("123")...)
 		v = append(v, []byte("123")...)
+		_, _ = k, v // to pass ineffassign
 
 		return nil
 	}); err != nil {
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 7cd36ffc..aca43981 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -146,7 +146,7 @@ The commands are:
     buckets     print a list of buckets
     check       verifies integrity of bolt database
     compact     copies a bolt database, compacting it in the process
-    dump        print a hexidecimal dump of a single page
+    dump        print a hexadecimal dump of a single page
     get         print the value of a key in a bucket
     info        print basic info
     keys        print a list of keys in a bucket

From 92ba45fa6d8d687c2fd5927476a6ccefb113934a Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Wed, 23 Aug 2017 17:07:50 -0700
Subject: [PATCH 30/59] README: explain purpose of bbolt fork

Fixes #31
---
 README.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ba31d467..43f5d544 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,14 @@
-Bolt
+bbolt
 ====
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/coreos/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/coreos/bbolt) [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
 
+bbolt is a fork of [Ben Johnson's][gh_ben] moribund [Bolt][bolt] key/value
+store. The purpose of this fork is to provide the Go community with an active
+maintenance and development target for Bolt; the goal is improved reliability
+and stability. bbolt includes bug fixes, performance enhancements, and features
+not found in Bolt while preserving backwards compatibility with the Bolt API.
+
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,
 fast, and reliable database for projects that don't require a full database
@@ -12,6 +18,8 @@ Since Bolt is meant to be used as such a low-level piece of functionality,
 simplicity is key. The API will be small and only focus on getting values
 and setting values. That's it.
 
+[gh_ben]: https://github.com/benbjohnson
+[bolt]: https://github.com/boltdb/bolt
 [hyc_symas]: https://twitter.com/hyc_symas
 [lmdb]: http://symas.com/mdb/
 
@@ -813,7 +821,7 @@ Here are a few things to note when evaluating and using Bolt:
 
 ## Reading the Source
 
-Bolt is a relatively small code base (<3KLOC) for an embedded, serializable,
+Bolt is a relatively small code base (<5KLOC) for an embedded, serializable,
 transactional key/value database so it can be a good starting point for people
 interested in how databases work.
 

From 69fa13f2f18c24cf3a9d5bb098fc895eec324fef Mon Sep 17 00:00:00 2001
From: Tyler Treat <ttreat31@gmail.com>
Date: Thu, 24 Aug 2017 09:35:24 -0700
Subject: [PATCH 31/59] Add NoSync field to Options

This allows enabling NoSync when you only have access to the Options
passed into Open() and not the returned DB (as is frequently the case
with libraries).
---
 db.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/db.go b/db.go
index 71933cb6..47303687 100644
--- a/db.go
+++ b/db.go
@@ -160,6 +160,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	if options == nil {
 		options = DefaultOptions
 	}
+	db.NoSync = options.NoSync
 	db.NoGrowSync = options.NoGrowSync
 	db.MmapFlags = options.MmapFlags
 	db.NoFreelistSync = options.NoFreelistSync
@@ -1008,6 +1009,11 @@ type Options struct {
 
 	// PageSize overrides the default OS page size.
 	PageSize int
+
+	// NoSync sets the initial value of DB.NoSync. Normally this can just be
+	// set directly on the DB itself when returned from Open(), but this option
+	// is useful in APIs which expose Options but not the underlying DB.
+	NoSync bool
 }
 
 // DefaultOptions represent the options used if nil options are passed into Open().

From e6a9c1db87685b6f81a269b087a144c02f7bdc13 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Tue, 29 Aug 2017 22:44:32 -0700
Subject: [PATCH 32/59] add coverage reports

Fixes #11
---
 Makefile  | 7 ++++---
 README.md | 4 +++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 937a892d..d3f6ed26 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,8 @@ errcheck:
 	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt
 
 test:
-	@go test -v -cover .
-	@go test -v ./cmd/bolt
+	go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	go test -v ./cmd/bolt
 
-.PHONY: fmt test
+.PHONY: race fmt errcheck test
diff --git a/README.md b/README.md
index ba31d467..6be2eaa1 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,9 @@
 Bolt
 ====
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/coreos/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/coreos/bbolt) [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
+[![Go Report Card](https://goreportcard.com/badge/github.com/coreos/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/coreos/bbolt)
+[![Coverage](https://codecov.io/gh/coreos/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/coreos/bbolt)
+[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,

From 3a49aacce1fe2ebf813046c7a25589988af6c37d Mon Sep 17 00:00:00 2001
From: Raphael Geronimi <raphael.geronimi@gmail.com>
Date: Wed, 6 Sep 2017 23:24:56 +0200
Subject: [PATCH 33/59] Added support for no timeout locks on db files (#35)

No longer unconditionally sleeps if file lock is already held
---
 bolt_unix.go         | 33 ++++++++++++++++++---------------
 bolt_unix_solaris.go | 39 +++++++++++++++++++--------------------
 bolt_windows.go      | 31 ++++++++++++++++---------------
 db.go                |  3 +++
 4 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/bolt_unix.go b/bolt_unix.go
index cad62dda..17be1e4e 100644
--- a/bolt_unix.go
+++ b/bolt_unix.go
@@ -13,29 +13,32 @@ import (
 // flock acquires an advisory lock on a file descriptor.
 func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+	        t = time.Now()
+	}
+	fd := db.file.Fd()
+	flag := syscall.LOCK_NB
+	if exclusive {
+		flag |= syscall.LOCK_EX
+	} else {
+		flag |= syscall.LOCK_SH 
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		flag := syscall.LOCK_SH
-		if exclusive {
-			flag = syscall.LOCK_EX
-		}
-
-		// Otherwise attempt to obtain an exclusive lock.
-		err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB)
+		// Attempt to obtain an exclusive lock.
+		err := syscall.Flock(int(fd), flag)
 		if err == nil {
 			return nil
 		} else if err != syscall.EWOULDBLOCK {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go
index 307bf2b3..c397f87d 100644
--- a/bolt_unix_solaris.go
+++ b/bolt_unix_solaris.go
@@ -13,34 +13,33 @@ import (
 // flock acquires an advisory lock on a file descriptor.
 func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+	        t = time.Now()
+	}
+	fd := db.file.Fd()
+	var lockType int16
+	if exclusive {
+		lockType = syscall.F_WRLCK
+	} else {
+		lockType = syscall.F_RDLCK
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		var lock syscall.Flock_t
-		lock.Start = 0
-		lock.Len = 0
-		lock.Pid = 0
-		lock.Whence = 0
-		lock.Pid = 0
-		if exclusive {
-			lock.Type = syscall.F_WRLCK
-		} else {
-			lock.Type = syscall.F_RDLCK
-		}
-		err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock)
+		// Attempt to obtain an exclusive lock.
+		lock := syscall.Flock_t{Type: lockType}
+		err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock)
 		if err == nil {
 			return nil
 		} else if err != syscall.EAGAIN {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
diff --git a/bolt_windows.go b/bolt_windows.go
index b00fb072..cad774cb 100644
--- a/bolt_windows.go
+++ b/bolt_windows.go
@@ -59,29 +59,30 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 	db.lockfile = f
 
 	var t time.Time
+	if timeout != 0 {
+	        t = time.Now()
+	}
+	fd := db.file.Fd()
+	var flag uint32 = flagLockFailImmediately
+	if exclusive {
+		flag |= flagLockExclusive
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-
-		var flag uint32 = flagLockFailImmediately
-		if exclusive {
-			flag |= flagLockExclusive
-		}
-
-		err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{})
+		// Attempt to obtain an exclusive lock.
+		err := lockFileEx(syscall.Handle(fd), flag, 0, 1, 0, &syscall.Overlapped{})
 		if err == nil {
 			return nil
 		} else if err != errLockViolation {
 			return err
 		}
 
+		// If we timed oumercit then return an error.
+		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
diff --git a/db.go b/db.go
index 47303687..e4475039 100644
--- a/db.go
+++ b/db.go
@@ -40,6 +40,9 @@ const (
 // default page size for db is set to the OS page size.
 var defaultPageSize = os.Getpagesize()
 
+// The time elapsed between consecutive file locking attempts.
+const flockRetryTimeout = 50 * time.Millisecond
+
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.

From 9245fa77322aa7734e7a57b78df907754d5acc52 Mon Sep 17 00:00:00 2001
From: Joe Betz <jpbetz@google.com>
Date: Thu, 7 Sep 2017 11:17:01 -0700
Subject: [PATCH 34/59] Increase freelist.releaseRange unit test coverage.

---
 freelist_test.go | 128 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)

diff --git a/freelist_test.go b/freelist_test.go
index 4259bedc..6b6c0ac3 100644
--- a/freelist_test.go
+++ b/freelist_test.go
@@ -44,6 +44,134 @@ func TestFreelist_release(t *testing.T) {
 	}
 }
 
+// Ensure that releaseRange handles boundary conditions correctly
+func TestFreelist_releaseRange(t *testing.T) {
+	type testRange struct {
+		begin, end txid
+	}
+
+	type testPage struct {
+		id       pgid
+		n        int
+		allocTxn txid
+		freeTxn  txid
+	}
+
+	var releaseRangeTests = []struct {
+		title         string
+		pagesIn       []testPage
+		releaseRanges []testRange
+		wantFree      []pgid
+	}{
+		{
+			title:         "Single pending in range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{1, 300}},
+			wantFree:      []pgid{3},
+		},
+		{
+			title:         "Single pending with minimum end range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{1, 200}},
+			wantFree:      []pgid{3},
+		},
+		{
+			title:         "Single pending outsize minimum end range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{1, 199}},
+			wantFree:      nil,
+		},
+		{
+			title:         "Single pending with minimum begin range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{100, 300}},
+			wantFree:      []pgid{3},
+		},
+		{
+			title:         "Single pending outside minimum begin range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{101, 300}},
+			wantFree:      nil,
+		},
+		{
+			title:         "Single pending in minimum range",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{199, 200}},
+			wantFree:      []pgid{3},
+		},
+		{
+			title:         "Single pending and read transaction at 199",
+			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
+			releaseRanges: []testRange{testRange{100, 198}, testRange{200, 300}},
+			wantFree:      nil,
+		},
+		{
+			title: "Adjacent pending and read transactions at 199, 200",
+			pagesIn: []testPage{
+				testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
+				testPage{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
+			},
+			releaseRanges: []testRange{
+				testRange{100, 198},
+				testRange{200, 199}, // Simulate the ranges db.freePages might produce.
+				testRange{201, 300},
+			},
+			wantFree: nil,
+		},
+		{
+			title: "Out of order ranges",
+			pagesIn: []testPage{
+				testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
+				testPage{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
+			},
+			releaseRanges: []testRange{
+				testRange{201, 199},
+				testRange{201, 200},
+				testRange{200, 200},
+			},
+			wantFree: nil,
+		},
+		{
+			title: "Multiple pending, read transaction at 150",
+			pagesIn: []testPage{
+				testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200},
+				testPage{id: 4, n: 1, allocTxn: 100, freeTxn: 125},
+				testPage{id: 5, n: 1, allocTxn: 125, freeTxn: 150},
+				testPage{id: 6, n: 1, allocTxn: 125, freeTxn: 175},
+				testPage{id: 7, n: 2, allocTxn: 150, freeTxn: 175},
+				testPage{id: 9, n: 2, allocTxn: 175, freeTxn: 200},
+			},
+			releaseRanges: []testRange{testRange{50, 149}, testRange{151, 300}},
+			wantFree:      []pgid{4, 9},
+		},
+	}
+
+	for _, c := range releaseRangeTests {
+		f := newFreelist()
+
+		for _, p := range c.pagesIn {
+			for i := uint64(0); i < uint64(p.n); i++ {
+				f.ids = append(f.ids, pgid(uint64(p.id)+i))
+			}
+		}
+		for _, p := range c.pagesIn {
+			f.allocate(p.allocTxn, p.n)
+		}
+
+		for _, p := range c.pagesIn {
+			f.free(p.freeTxn, &page{id: p.id})
+		}
+
+		for _, r := range c.releaseRanges {
+			f.releaseRange(r.begin, r.end)
+		}
+
+		if exp := c.wantFree; !reflect.DeepEqual(exp, f.ids) {
+			t.Errorf("exp=%v; got=%v for %s", exp, f.ids, c.title)
+		}
+	}
+}
+
 // Ensure that a freelist can find contiguous blocks of pages.
 func TestFreelist_allocate(t *testing.T) {
 	f := newFreelist()

From 4ce1b5e57946942dbf1f838431af055b359bf6a2 Mon Sep 17 00:00:00 2001
From: lorneli <lorneli@163.com>
Date: Mon, 11 Sep 2017 12:53:25 +0800
Subject: [PATCH 35/59] tx: use io.SeekStart in WriteTo function

Const os.SEEK_SET is deprecated.
---
 tx.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tx.go b/tx.go
index 57c4d5ad..cee8745e 100644
--- a/tx.go
+++ b/tx.go
@@ -345,7 +345,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 	}
 
 	// Move past the meta pages in the file.
-	if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil {
+	if _, err := f.Seek(int64(tx.db.pageSize*2), io.SeekStart); err != nil {
 		return n, fmt.Errorf("seek: %s", err)
 	}
 

From d72f7607a6ab46c709482efa6c674194061b513d Mon Sep 17 00:00:00 2001
From: Joe Betz <jpbetz@google.com>
Date: Thu, 7 Sep 2017 16:11:04 -0700
Subject: [PATCH 36/59] Improve test coverage for releaseRange rollbacks.

---
 tx_test.go | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/tx_test.go b/tx_test.go
index ff512b19..7b71f5b5 100644
--- a/tx_test.go
+++ b/tx_test.go
@@ -602,6 +602,111 @@ func TestTx_CopyFile_Error_Normal(t *testing.T) {
 	}
 }
 
+// TestTx_releaseRange ensures db.freePages handles page releases
+// correctly when there are transaction that are no longer reachable
+// via any read/write transactions and are "between" ongoing read
+// transactions, which requires they must be freed by
+// freelist.releaseRange.
+func TestTx_releaseRange(t *testing.T) {
+	// Set initial mmap size well beyond the limit we will hit in this
+	// test, since we are testing with long running read transactions
+	// and will deadlock if db.grow is triggered.
+	db := MustOpenWithOption(&bolt.Options{InitialMmapSize: os.Getpagesize() * 100})
+	defer db.MustClose()
+
+	bucket := "bucket"
+
+	put := func(key, value string) {
+		if err := db.Update(func(tx *bolt.Tx) error {
+			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
+			if err != nil {
+				t.Fatal(err)
+			}
+			return b.Put([]byte(key), []byte(value))
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	del := func(key string) {
+		if err := db.Update(func(tx *bolt.Tx) error {
+			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
+			if err != nil {
+				t.Fatal(err)
+			}
+			return b.Delete([]byte(key))
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	getWithTxn := func(txn *bolt.Tx, key string) []byte {
+		return txn.Bucket([]byte(bucket)).Get([]byte(key))
+	}
+
+	openReadTxn := func() *bolt.Tx {
+		readTx, err := db.Begin(false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return readTx
+	}
+
+	checkWithReadTxn := func(txn *bolt.Tx, key string, wantValue []byte) {
+		value := getWithTxn(txn, key)
+		if !bytes.Equal(value, wantValue) {
+			t.Errorf("Wanted value to be %s for key %s, but got %s", wantValue, key, string(value))
+		}
+	}
+
+	rollback := func(txn *bolt.Tx) {
+		if err := txn.Rollback(); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	put("k1", "v1")
+	rtx1 := openReadTxn()
+	put("k2", "v2")
+	hold1 := openReadTxn()
+	put("k3", "v3")
+	hold2 := openReadTxn()
+	del("k3")
+	rtx2 := openReadTxn()
+	del("k1")
+	hold3 := openReadTxn()
+	del("k2")
+	hold4 := openReadTxn()
+	put("k4", "v4")
+	hold5 := openReadTxn()
+
+	// Close the read transactions we established to hold a portion of the pages in pending state.
+	rollback(hold1)
+	rollback(hold2)
+	rollback(hold3)
+	rollback(hold4)
+	rollback(hold5)
+
+	// Execute a write transaction to trigger a releaseRange operation in the db
+	// that will free multiple ranges between the remaining open read transactions, now that the
+	// holds have been rolled back.
+	put("k4", "v4")
+
+	// Check that all long running reads still read correct values.
+	checkWithReadTxn(rtx1, "k1", []byte("v1"))
+	checkWithReadTxn(rtx2, "k2", []byte("v2"))
+	rollback(rtx1)
+	rollback(rtx2)
+
+	// Check that the final state is correct.
+	rtx7 := openReadTxn()
+	checkWithReadTxn(rtx7, "k1", nil)
+	checkWithReadTxn(rtx7, "k2", nil)
+	checkWithReadTxn(rtx7, "k3", nil)
+	checkWithReadTxn(rtx7, "k4", []byte("v4"))
+	rollback(rtx7)
+}
+
 func ExampleTx_Rollback() {
 	// Open the database.
 	db, err := bolt.Open(tempfile(), 0666, nil)

From d294ec8a4251296644cbe25531f570d40fdd497f Mon Sep 17 00:00:00 2001
From: lorneli <lorneli@163.com>
Date: Tue, 12 Sep 2017 20:17:59 +0800
Subject: [PATCH 37/59] db: add test in read-only mode

---
 db_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/db_test.go b/db_test.go
index b364017e..64a59dde 100644
--- a/db_test.go
+++ b/db_test.go
@@ -415,6 +415,59 @@ func TestDB_Open_InitialMmapSize(t *testing.T) {
 	}
 }
 
+// TestDB_Open_ReadOnly checks a database in read only mode can read but not write.
+func TestDB_Open_ReadOnly(t *testing.T) {
+	// Create a writable db, write k-v and close it.
+	db := MustOpenDB()
+	defer db.Close()
+	if err := db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucket([]byte("widgets"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := b.Put([]byte("foo"), []byte("bar")); err != nil {
+			t.Fatal(err)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	f := db.f
+	o := &bolt.Options{ReadOnly: true}
+	readOnlyDB, err := bolt.Open(f, 0666, o)
+	if err != nil {
+		panic(err)
+	}
+
+	if !readOnlyDB.IsReadOnly() {
+		t.Fatal("expect db in read only mode")
+	}
+
+	// Read from a read-only transaction.
+	if err := readOnlyDB.View(func(tx *bolt.Tx) error {
+		value := tx.Bucket([]byte("widgets")).Get([]byte("foo"))
+		if bytes.Compare(value, []byte("bar")) != 0 {
+			t.Fatal("expect value 'bar', got", value)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Can't launch read-write transaction.
+	if _, err := readOnlyDB.Begin(true); err != bolt.ErrDatabaseReadOnly {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if err := readOnlyDB.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // TestOpen_BigPage checks the database uses bigger pages when
 // changing PageSize.
 func TestOpen_BigPage(t *testing.T) {
@@ -1449,15 +1502,7 @@ type DB struct {
 
 // MustOpenDB returns a new, open DB at a temporary location.
 func MustOpenDB() *DB {
-	f := tempfile()
-	db, err := bolt.Open(f, 0666, nil)
-	if err != nil {
-		panic(err)
-	}
-	return &DB{
-		DB: db,
-		f:  f,
-	}
+	return MustOpenWithOption(nil)
 }
 
 // MustOpenDBWithOption returns a new, open DB at a temporary location with given options.

From ea18f34f9d3f308b587515399fc5dd1af69c5379 Mon Sep 17 00:00:00 2001
From: lorneli <lorneli@163.com>
Date: Sun, 10 Sep 2017 14:53:37 +0800
Subject: [PATCH 38/59] db: return t.Rollback directly in the end of View
 function

Make return line of db.View corresponding with db.Update.
---
 db.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/db.go b/db.go
index e4475039..6559f416 100644
--- a/db.go
+++ b/db.go
@@ -687,11 +687,7 @@ func (db *DB) View(fn func(*Tx) error) error {
 		return err
 	}
 
-	if err := t.Rollback(); err != nil {
-		return err
-	}
-
-	return nil
+	return t.Rollback()
 }
 
 // Batch calls fn as part of a batch. It behaves similar to Update,

From 53a930f1e1d735283e3a839ee30978f08279ed90 Mon Sep 17 00:00:00 2001
From: lorneli <lorneli@163.com>
Date: Mon, 11 Sep 2017 20:41:08 +0800
Subject: [PATCH 39/59] tx: just close file once in WriteTo function

WriteTo function closes file twice in normal path previously.
---
 tx.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tx.go b/tx.go
index 57c4d5ad..3c4b144c 100644
--- a/tx.go
+++ b/tx.go
@@ -317,7 +317,11 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 	if err != nil {
 		return 0, err
 	}
-	defer func() { _ = f.Close() }()
+	defer func() {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}()
 
 	// Generate a meta page. We use the same page data for both meta pages.
 	buf := make([]byte, tx.db.pageSize)
@@ -356,7 +360,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 		return n, err
 	}
 
-	return n, f.Close()
+	return n, nil
 }
 
 // CopyFile copies the entire database to file at the given path.

From e39821f3defb7f31a52ff48f5e1b01433dd713a3 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Wed, 13 Sep 2017 13:27:42 -0700
Subject: [PATCH 40/59] *: fix gofmt errors and makefile test

---
 Makefile             | 13 ++++++----
 bolt_unix.go         |  6 ++---
 bolt_unix_solaris.go |  4 +--
 bolt_windows.go      |  4 +--
 db_test.go           |  2 +-
 freelist_test.go     | 62 ++++++++++++++++++++++----------------------
 6 files changed, 47 insertions(+), 44 deletions(-)

diff --git a/Makefile b/Makefile
index d3f6ed26..43b94f3b 100644
--- a/Makefile
+++ b/Makefile
@@ -7,13 +7,16 @@ default: build
 race:
 	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
 
-# go get honnef.co/go/tools/simple
-# go get honnef.co/go/tools/unused
 fmt:
+	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
+
+# go get honnef.co/go/tools/simple
+gosimple:
 	gosimple ./...
-	unused ./...
-	gofmt -l -s -d $(find -name \*.go)
 
+# go get honnef.co/go/tools/unused
+unused:
+	unused ./...
 
 # go get github.com/kisielk/errcheck
 errcheck:
@@ -24,4 +27,4 @@ test:
 	# Note: gets "program not an importable package" in out of path builds
 	go test -v ./cmd/bolt
 
-.PHONY: race fmt errcheck test
+.PHONY: race fmt errcheck test gosimple unused
diff --git a/bolt_unix.go b/bolt_unix.go
index 17be1e4e..06592a08 100644
--- a/bolt_unix.go
+++ b/bolt_unix.go
@@ -14,14 +14,14 @@ import (
 func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
 	var t time.Time
 	if timeout != 0 {
-	        t = time.Now()
+		t = time.Now()
 	}
 	fd := db.file.Fd()
 	flag := syscall.LOCK_NB
 	if exclusive {
 		flag |= syscall.LOCK_EX
 	} else {
-		flag |= syscall.LOCK_SH 
+		flag |= syscall.LOCK_SH
 	}
 	for {
 		// Attempt to obtain an exclusive lock.
@@ -33,7 +33,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 		}
 
 		// If we timed out then return an error.
-		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
 			return ErrTimeout
 		}
 
diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go
index c397f87d..fd8335ec 100644
--- a/bolt_unix_solaris.go
+++ b/bolt_unix_solaris.go
@@ -14,7 +14,7 @@ import (
 func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
 	var t time.Time
 	if timeout != 0 {
-	        t = time.Now()
+		t = time.Now()
 	}
 	fd := db.file.Fd()
 	var lockType int16
@@ -34,7 +34,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 		}
 
 		// If we timed out then return an error.
-		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
 			return ErrTimeout
 		}
 
diff --git a/bolt_windows.go b/bolt_windows.go
index cad774cb..77b66f8b 100644
--- a/bolt_windows.go
+++ b/bolt_windows.go
@@ -60,7 +60,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 
 	var t time.Time
 	if timeout != 0 {
-	        t = time.Now()
+		t = time.Now()
 	}
 	fd := db.file.Fd()
 	var flag uint32 = flagLockFailImmediately
@@ -77,7 +77,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 		}
 
 		// If we timed oumercit then return an error.
-		if timeout != 0 && time.Since(t) > timeout - flockRetryTimeout {
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
 			return ErrTimeout
 		}
 
diff --git a/db_test.go b/db_test.go
index 64a59dde..bd4927b4 100644
--- a/db_test.go
+++ b/db_test.go
@@ -450,7 +450,7 @@ func TestDB_Open_ReadOnly(t *testing.T) {
 	// Read from a read-only transaction.
 	if err := readOnlyDB.View(func(tx *bolt.Tx) error {
 		value := tx.Bucket([]byte("widgets")).Get([]byte("foo"))
-		if bytes.Compare(value, []byte("bar")) != 0 {
+		if !bytes.Equal(value, []byte("bar")) {
 			t.Fatal("expect value 'bar', got", value)
 		}
 		return nil
diff --git a/freelist_test.go b/freelist_test.go
index 6b6c0ac3..24ed4cf1 100644
--- a/freelist_test.go
+++ b/freelist_test.go
@@ -65,83 +65,83 @@ func TestFreelist_releaseRange(t *testing.T) {
 	}{
 		{
 			title:         "Single pending in range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{1, 300}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{{1, 300}},
 			wantFree:      []pgid{3},
 		},
 		{
 			title:         "Single pending with minimum end range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{1, 200}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{{1, 200}},
 			wantFree:      []pgid{3},
 		},
 		{
 			title:         "Single pending outsize minimum end range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{1, 199}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{{1, 199}},
 			wantFree:      nil,
 		},
 		{
 			title:         "Single pending with minimum begin range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{100, 300}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{{100, 300}},
 			wantFree:      []pgid{3},
 		},
 		{
 			title:         "Single pending outside minimum begin range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{101, 300}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 100, freeTxn: 200}},
+			releaseRanges: []testRange{{101, 300}},
 			wantFree:      nil,
 		},
 		{
 			title:         "Single pending in minimum range",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{199, 200}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
+			releaseRanges: []testRange{{199, 200}},
 			wantFree:      []pgid{3},
 		},
 		{
 			title:         "Single pending and read transaction at 199",
-			pagesIn:       []testPage{testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
-			releaseRanges: []testRange{testRange{100, 198}, testRange{200, 300}},
+			pagesIn:       []testPage{{id: 3, n: 1, allocTxn: 199, freeTxn: 200}},
+			releaseRanges: []testRange{{100, 198}, {200, 300}},
 			wantFree:      nil,
 		},
 		{
 			title: "Adjacent pending and read transactions at 199, 200",
 			pagesIn: []testPage{
-				testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
-				testPage{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
+				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
+				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
 			},
 			releaseRanges: []testRange{
-				testRange{100, 198},
-				testRange{200, 199}, // Simulate the ranges db.freePages might produce.
-				testRange{201, 300},
+				{100, 198},
+				{200, 199}, // Simulate the ranges db.freePages might produce.
+				{201, 300},
 			},
 			wantFree: nil,
 		},
 		{
 			title: "Out of order ranges",
 			pagesIn: []testPage{
-				testPage{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
-				testPage{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
+				{id: 3, n: 1, allocTxn: 199, freeTxn: 200},
+				{id: 4, n: 1, allocTxn: 200, freeTxn: 201},
 			},
 			releaseRanges: []testRange{
-				testRange{201, 199},
-				testRange{201, 200},
-				testRange{200, 200},
+				{201, 199},
+				{201, 200},
+				{200, 200},
 			},
 			wantFree: nil,
 		},
 		{
 			title: "Multiple pending, read transaction at 150",
 			pagesIn: []testPage{
-				testPage{id: 3, n: 1, allocTxn: 100, freeTxn: 200},
-				testPage{id: 4, n: 1, allocTxn: 100, freeTxn: 125},
-				testPage{id: 5, n: 1, allocTxn: 125, freeTxn: 150},
-				testPage{id: 6, n: 1, allocTxn: 125, freeTxn: 175},
-				testPage{id: 7, n: 2, allocTxn: 150, freeTxn: 175},
-				testPage{id: 9, n: 2, allocTxn: 175, freeTxn: 200},
+				{id: 3, n: 1, allocTxn: 100, freeTxn: 200},
+				{id: 4, n: 1, allocTxn: 100, freeTxn: 125},
+				{id: 5, n: 1, allocTxn: 125, freeTxn: 150},
+				{id: 6, n: 1, allocTxn: 125, freeTxn: 175},
+				{id: 7, n: 2, allocTxn: 150, freeTxn: 175},
+				{id: 9, n: 2, allocTxn: 175, freeTxn: 200},
 			},
-			releaseRanges: []testRange{testRange{50, 149}, testRange{151, 300}},
+			releaseRanges: []testRange{{50, 149}, {151, 300}},
 			wantFree:      []pgid{4, 9},
 		},
 	}

From ba5a58dde013b339b8f0f246bbef5c6424e6eb8d Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Wed, 13 Sep 2017 12:58:06 -0700
Subject: [PATCH 41/59] test tx.Check() on read only db

---
 tx_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/tx_test.go b/tx_test.go
index 7b71f5b5..de92cb53 100644
--- a/tx_test.go
+++ b/tx_test.go
@@ -11,6 +11,54 @@ import (
 	"github.com/coreos/bbolt"
 )
 
+// TestTx_Check_ReadOnly tests consistency checking on a ReadOnly database.
+func TestTx_Check_ReadOnly(t *testing.T) {
+	db := MustOpenDB()
+	defer db.Close()
+	if err := db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucket([]byte("widgets"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := b.Put([]byte("foo"), []byte("bar")); err != nil {
+			t.Fatal(err)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	readOnlyDB, err := bolt.Open(db.f, 0666, &bolt.Options{ReadOnly: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer readOnlyDB.Close()
+
+	tx, err := readOnlyDB.Begin(false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// ReadOnly DB will load freelist on Check call.
+	numChecks := 2
+	errc := make(chan error, numChecks)
+	check := func() {
+		err, _ := <-tx.Check()
+		errc <- err
+	}
+	// Ensure the freelist is not reloaded and does not race.
+	for i := 0; i < numChecks; i++ {
+		go check()
+	}
+	for i := 0; i < numChecks; i++ {
+		if err := <-errc; err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
 // Ensure that committing a closed transaction returns an error.
 func TestTx_Commit_ErrTxClosed(t *testing.T) {
 	db := MustOpenDB()

From 69918b9e4e96a2a2733acf3d60b15143425e396e Mon Sep 17 00:00:00 2001
From: Joe Betz <jpbetz@google.com>
Date: Wed, 13 Sep 2017 23:06:49 -0700
Subject: [PATCH 42/59] Fix bolt CLI tool print entire freelist, and to dump
 keys/value bytes of leaf elements.

---
 .gitignore       |   1 +
 cmd/bolt/main.go | 186 +++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 182 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index c7bd2b7a..c2a8cfa7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 *.test
 *.swp
 /bin/
+cmd/bolt/bolt
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index aca43981..eb85e05c 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -114,6 +114,8 @@ func (m *Main) Run(args ...string) error {
 		return newCompactCommand(m).Run(args[1:]...)
 	case "dump":
 		return newDumpCommand(m).Run(args[1:]...)
+	case "page-item":
+		return newPageItemCommand(m).Run(args[1:]...)
 	case "get":
 		return newGetCommand(m).Run(args[1:]...)
 	case "info":
@@ -153,6 +155,7 @@ The commands are:
     help        print this screen
     page        print one or more pages in human readable format
     pages       print list of pages with their types
+    page-item   print the key and value of a page item.
     stats       iterate over all pages and generate usage stats
 
 Use "bolt [command] -h" for more information about a command.
@@ -416,9 +419,173 @@ func (cmd *DumpCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSi
 // Usage returns the help message.
 func (cmd *DumpCommand) Usage() string {
 	return strings.TrimLeft(`
-usage: bolt dump -page PAGEID PATH
+usage: bolt dump PATH pageid [pageid...]
 
-Dump prints a hexadecimal dump of a single page.
+Dump prints a hexadecimal dump of one or more pages.
+`, "\n")
+}
+
+// PageItemCommand represents the "page-item" command execution.
+type PageItemCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// newPageItemCommand returns a PageItemCommand.
+func newPageItemCommand(m *Main) *PageItemCommand {
+	return &PageItemCommand{
+		Stdin:  m.Stdin,
+		Stdout: m.Stdout,
+		Stderr: m.Stderr,
+	}
+}
+
+type pageItemOptions struct {
+	help      bool
+	keyOnly   bool
+	valueOnly bool
+	format    string
+}
+
+// Run executes the command.
+func (cmd *PageItemCommand) Run(args ...string) error {
+	// Parse flags.
+	options := &pageItemOptions{}
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.BoolVar(&options.keyOnly, "key-only", false, "Print only the key")
+	fs.BoolVar(&options.valueOnly, "value-only", false, "Print only the value")
+	fs.StringVar(&options.format, "format", "ascii-encoded", "Output format. One of: ascii-encoded|hex|bytes")
+	fs.BoolVar(&options.help, "h", false, "")
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if options.help {
+		fmt.Fprintln(cmd.Stderr, cmd.Usage())
+		return ErrUsage
+	}
+
+	if options.keyOnly && options.valueOnly {
+		return fmt.Errorf("The --key-only or --value-only flag may be set, but not both.")
+	}
+
+	// Require database path and page id.
+	path := fs.Arg(0)
+	if path == "" {
+		return ErrPathRequired
+	} else if _, err := os.Stat(path); os.IsNotExist(err) {
+		return ErrFileNotFound
+	}
+
+	// Read page id.
+	pageID, err := strconv.Atoi(fs.Arg(1))
+	if err != nil {
+		return err
+	}
+
+	// Read item id.
+	itemID, err := strconv.Atoi(fs.Arg(2))
+	if err != nil {
+		return err
+	}
+
+	// Open database file handler.
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }()
+
+	// Retrieve page info and page size.
+	_, buf, err := ReadPage(path, pageID)
+	if err != nil {
+		return err
+	}
+
+	if !options.valueOnly {
+		err := cmd.PrintLeafItemKey(cmd.Stdout, buf, uint16(itemID), options.format)
+		if err != nil {
+			return err
+		}
+	}
+	if !options.keyOnly {
+		err := cmd.PrintLeafItemValue(cmd.Stdout, buf, uint16(itemID), options.format)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// leafPageElement retrieves a leaf page element.
+func (cmd *PageItemCommand) leafPageElement(pageBytes []byte, index uint16) (*leafPageElement, error) {
+	p := (*page)(unsafe.Pointer(&pageBytes[0]))
+	if index >= p.count {
+		return nil, fmt.Errorf("leafPageElement: expected item index less than %d, but got %d.", p.count, index)
+	}
+	if p.Type() != "leaf" {
+		return nil, fmt.Errorf("leafPageElement: expected page type of 'leaf', but got '%s'", p.Type())
+	}
+	return p.leafPageElement(index), nil
+}
+
+// writeBytes writes the byte to the writer. Supported formats: ascii-encoded, hex, bytes.
+func (cmd *PageItemCommand) writeBytes(w io.Writer, b []byte, format string) error {
+	switch format {
+	case "ascii-encoded":
+		_, err := fmt.Fprintf(w, "%q", b)
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprintf(w, "\n")
+		return err
+	case "hex":
+		_, err := fmt.Fprintf(w, "%x", b)
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprintf(w, "\n")
+		return err
+	case "bytes":
+		_, err := w.Write(b)
+		return err
+	default:
+		return fmt.Errorf("writeBytes: unsupported format: %s", format)
+	}
+}
+
+// PrintLeafItemKey writes the bytes of a leaf element's key.
+func (cmd *PageItemCommand) PrintLeafItemKey(w io.Writer, pageBytes []byte, index uint16, format string) error {
+	e, err := cmd.leafPageElement(pageBytes, index)
+	if err != nil {
+		return err
+	}
+	return cmd.writeBytes(w, e.key(), format)
+}
+
+// PrintLeafItemKey writes the bytes of a leaf element's value.
+func (cmd *PageItemCommand) PrintLeafItemValue(w io.Writer, pageBytes []byte, index uint16, format string) error {
+	e, err := cmd.leafPageElement(pageBytes, index)
+	if err != nil {
+		return err
+	}
+	return cmd.writeBytes(w, e.value(), format)
+}
+
+// Usage returns the help message.
+func (cmd *PageItemCommand) Usage() string {
+	return strings.TrimLeft(`
+usage: bolt page-item [options] PATH pageid itemid
+
+Additional options include:
+
+	--key-only
+		Print only the key
+	--value-only
+		Print only the value
+	--format
+		Output format. One of: ascii-encoded|hex|bytes (default=ascii-encoded)
+
+page-item prints a page item key and value.
 `, "\n")
 }
 
@@ -592,13 +759,22 @@ func (cmd *PageCommand) PrintBranch(w io.Writer, buf []byte) error {
 func (cmd *PageCommand) PrintFreelist(w io.Writer, buf []byte) error {
 	p := (*page)(unsafe.Pointer(&buf[0]))
 
+	// Check for overflow and, if present, adjust starting index and actual element count.
+	idx, count := 0, int(p.count)
+	if p.count == 0xFFFF {
+		idx = 1
+		count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0])
+	}
+
 	// Print number of items.
-	fmt.Fprintf(w, "Item Count: %d\n", p.count)
+	fmt.Fprintf(w, "Item Count: %d\n", count)
+	fmt.Fprintf(w, "Overflow: %d\n", p.overflow)
+
 	fmt.Fprintf(w, "\n")
 
 	// Print each page in the freelist.
 	ids := (*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr))
-	for i := uint16(0); i < p.count; i++ {
+	for i := int(idx); i < count; i++ {
 		fmt.Fprintf(w, "%d\n", ids[i])
 	}
 	fmt.Fprintf(w, "\n")
@@ -653,7 +829,7 @@ func (cmd *PageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSi
 // Usage returns the help message.
 func (cmd *PageCommand) Usage() string {
 	return strings.TrimLeft(`
-usage: bolt page -page PATH pageid [pageid...]
+usage: bolt page PATH pageid [pageid...]
 
 Page prints one or more pages in human readable format.
 `, "\n")

From bdfe4158f8eb42af576f56cb2ff57fd70bb88a3b Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Wed, 13 Sep 2017 13:16:42 -0700
Subject: [PATCH 43/59] tx: load freelist on Check()

Otherwise, nil dereference on ReadOnly DB

Fixes #45
---
 db.go | 43 +++++++++++++++++++++++++++++--------------
 tx.go |  3 +++
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/db.go b/db.go
index 6559f416..08dfd0ab 100644
--- a/db.go
+++ b/db.go
@@ -116,9 +116,11 @@ type DB struct {
 	opened   bool
 	rwtx     *Tx
 	txs      []*Tx
-	freelist *freelist
 	stats    Stats
 
+	freelist     *freelist
+	freelistLoad sync.Once
+
 	pagePool sync.Pool
 
 	batchMu sync.Mutex
@@ -157,8 +159,9 @@ func (db *DB) String() string {
 // If the file does not exist then it will be created automatically.
 // Passing in nil options will cause Bolt to open the database with the default options.
 func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
-	var db = &DB{opened: true}
-
+	db := &DB{
+		opened: true,
+	}
 	// Set default options if no options are provided.
 	if options == nil {
 		options = DefaultOptions
@@ -254,20 +257,11 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return db, nil
 	}
 
-	db.freelist = newFreelist()
-	noFreeList := db.meta().freelist == pgidNoFreelist
-	if noFreeList {
-		// Reconstruct free list by scanning the DB.
-		db.freelist.readIDs(db.freepages())
-	} else {
-		// Read free list from freelist page.
-		db.freelist.read(db.page(db.meta().freelist))
-	}
-	db.stats.FreePageN = len(db.freelist.ids)
+	db.loadFreelist()
 
 	// Flush freelist when transitioning from no sync to sync so
 	// NoFreelistSync unaware boltdb can open the db later.
-	if !db.NoFreelistSync && noFreeList {
+	if !db.NoFreelistSync && !db.hasSyncedFreelist() {
 		tx, err := db.Begin(true)
 		if tx != nil {
 			err = tx.Commit()
@@ -282,6 +276,27 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	return db, nil
 }
 
+// loadFreelist reads the freelist if it is synced, or reconstructs it
+// by scanning the DB if it is not synced. It assumes there are no
+// concurrent accesses being made to the freelist.
+func (db *DB) loadFreelist() {
+	db.freelistLoad.Do(func() {
+		db.freelist = newFreelist()
+		if !db.hasSyncedFreelist() {
+			// Reconstruct free list by scanning the DB.
+			db.freelist.readIDs(db.freepages())
+		} else {
+			// Read free list from freelist page.
+			db.freelist.read(db.page(db.meta().freelist))
+		}
+		db.stats.FreePageN = len(db.freelist.ids)
+	})
+}
+
+func (db *DB) hasSyncedFreelist() bool {
+	return db.meta().freelist != pgidNoFreelist
+}
+
 // mmap opens the underlying memory-mapped file and initializes the meta references.
 // minsz is the minimum size that the new mmap can be.
 func (db *DB) mmap(minsz int) error {
diff --git a/tx.go b/tx.go
index e6d95ca6..aefcec09 100644
--- a/tx.go
+++ b/tx.go
@@ -395,6 +395,9 @@ func (tx *Tx) Check() <-chan error {
 }
 
 func (tx *Tx) check(ch chan error) {
+	// Force loading free list if opened in ReadOnly mode.
+	tx.db.loadFreelist()
+
 	// Check if any pages are double freed.
 	freed := make(map[pgid]bool)
 	all := make([]pgid, tx.db.freelist.count())

From 235a4273ef6b04c8406d7284b80feee3c4f39226 Mon Sep 17 00:00:00 2001
From: Sue Spence <virtuallysue@gmail.com>
Date: Mon, 25 Sep 2017 13:58:09 +0100
Subject: [PATCH 44/59] Removed 'moribund' since some people think it's a bit
 derogatory towards the original developer.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4e172928..99052aa1 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@ bbolt
 [![Coverage](https://codecov.io/gh/coreos/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/coreos/bbolt)
 [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt)
 
-bbolt is a fork of [Ben Johnson's][gh_ben] moribund [Bolt][bolt] key/value
+bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value
 store. The purpose of this fork is to provide the Go community with an active
 maintenance and development target for Bolt; the goal is improved reliability
 and stability. bbolt includes bug fixes, performance enhancements, and features

From 68861c5f876d1ac006f6cfa93c8076d17da29bd5 Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Thu, 28 Sep 2017 01:39:15 -0700
Subject: [PATCH 45/59] db_test.go: remove temp files after tests

Was filling up all the disk space in Jenkins VMs.

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 db_test.go | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/db_test.go b/db_test.go
index bd4927b4..629150c9 100644
--- a/db_test.go
+++ b/db_test.go
@@ -44,6 +44,8 @@ type meta struct {
 // Ensure that a database can be opened without error.
 func TestOpen(t *testing.T) {
 	path := tempfile()
+	defer os.RemoveAll(path)
+
 	db, err := bolt.Open(path, 0666, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -79,6 +81,7 @@ func TestOpen_ErrNotExists(t *testing.T) {
 // Ensure that opening a file that is not a Bolt database returns ErrInvalid.
 func TestOpen_ErrInvalid(t *testing.T) {
 	path := tempfile()
+	defer os.RemoveAll(path)
 
 	f, err := os.Create(path)
 	if err != nil {
@@ -90,7 +93,6 @@ func TestOpen_ErrInvalid(t *testing.T) {
 	if err := f.Close(); err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(path)
 
 	if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrInvalid {
 		t.Fatalf("unexpected error: %s", err)
@@ -302,15 +304,16 @@ func TestOpen_Size_Large(t *testing.T) {
 // Ensure that a re-opened database is consistent.
 func TestOpen_Check(t *testing.T) {
 	path := tempfile()
+	defer os.RemoveAll(path)
 
 	db, err := bolt.Open(path, 0666, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil {
+	if err = db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil {
 		t.Fatal(err)
 	}
-	if err := db.Close(); err != nil {
+	if err = db.Close(); err != nil {
 		t.Fatal(err)
 	}
 
@@ -334,18 +337,19 @@ func TestOpen_MetaInitWriteError(t *testing.T) {
 // Ensure that a database that is too small returns an error.
 func TestOpen_FileTooSmall(t *testing.T) {
 	path := tempfile()
+	defer os.RemoveAll(path)
 
 	db, err := bolt.Open(path, 0666, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	pageSize := int64(db.Info().PageSize)
-	if err := db.Close(); err != nil {
+	if err = db.Close(); err != nil {
 		t.Fatal(err)
 	}
 
 	// corrupt the database
-	if err := os.Truncate(path, pageSize); err != nil {
+	if err = os.Truncate(path, pageSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -419,7 +423,8 @@ func TestDB_Open_InitialMmapSize(t *testing.T) {
 func TestDB_Open_ReadOnly(t *testing.T) {
 	// Create a writable db, write k-v and close it.
 	db := MustOpenDB()
-	defer db.Close()
+	defer db.MustClose()
+
 	if err := db.Update(func(tx *bolt.Tx) error {
 		b, err := tx.CreateBucket([]byte("widgets"))
 		if err != nil {
@@ -507,7 +512,7 @@ func TestOpen_RecoverFreeList(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	if err := tx.Commit(); err != nil {
+	if err = tx.Commit(); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1270,7 +1275,7 @@ func ExampleDB_Begin_ReadOnly() {
 	defer os.Remove(db.Path())
 
 	// Create a bucket using a read-write transaction.
-	if err := db.Update(func(tx *bolt.Tx) error {
+	if err = db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucket([]byte("widgets"))
 		return err
 	}); err != nil {
@@ -1283,16 +1288,16 @@ func ExampleDB_Begin_ReadOnly() {
 		log.Fatal(err)
 	}
 	b := tx.Bucket([]byte("widgets"))
-	if err := b.Put([]byte("john"), []byte("blue")); err != nil {
+	if err = b.Put([]byte("john"), []byte("blue")); err != nil {
 		log.Fatal(err)
 	}
-	if err := b.Put([]byte("abby"), []byte("red")); err != nil {
+	if err = b.Put([]byte("abby"), []byte("red")); err != nil {
 		log.Fatal(err)
 	}
-	if err := b.Put([]byte("zephyr"), []byte("purple")); err != nil {
+	if err = b.Put([]byte("zephyr"), []byte("purple")); err != nil {
 		log.Fatal(err)
 	}
-	if err := tx.Commit(); err != nil {
+	if err = tx.Commit(); err != nil {
 		log.Fatal(err)
 	}
 
@@ -1306,11 +1311,11 @@ func ExampleDB_Begin_ReadOnly() {
 		fmt.Printf("%s likes %s\n", k, v)
 	}
 
-	if err := tx.Rollback(); err != nil {
+	if err = tx.Rollback(); err != nil {
 		log.Fatal(err)
 	}
 
-	if err := db.Close(); err != nil {
+	if err = db.Close(); err != nil {
 		log.Fatal(err)
 	}
 

From b4c13d4814025e7e325d1b6f2f57810c37a3bc42 Mon Sep 17 00:00:00 2001
From: spacewander <spacewanderlzx@gmail.com>
Date: Sun, 29 Oct 2017 16:55:51 +0800
Subject: [PATCH 46/59] Add 'boltcli' to the list of projects that use Bolt

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 99052aa1..015f0efb 100644
--- a/README.md
+++ b/README.md
@@ -919,6 +919,7 @@ Below is a list of public, open source projects that use Bolt:
 * [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development.
 * [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains
 * [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal.
+* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support.
 * [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet.
 * [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency.
 * [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies

From 434419a2a008804109fe5bba69e032e9d4dc9c7c Mon Sep 17 00:00:00 2001
From: kwf2030 <wangfeng@supertu.cn>
Date: Wed, 1 Nov 2017 08:55:55 +0800
Subject: [PATCH 47/59] fix funlock error when call db.Close on windows

---
 bolt_windows.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bolt_windows.go b/bolt_windows.go
index 77b66f8b..ca6f9a11 100644
--- a/bolt_windows.go
+++ b/bolt_windows.go
@@ -62,7 +62,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
 	if timeout != 0 {
 		t = time.Now()
 	}
-	fd := db.file.Fd()
+	fd := f.Fd()
 	var flag uint32 = flagLockFailImmediately
 	if exclusive {
 		flag |= flagLockExclusive

From 22635d7451281821e01aa786b7e605e8982d0b99 Mon Sep 17 00:00:00 2001
From: "zhesi.huang" <huangzhesi@gamil.com>
Date: Thu, 9 Nov 2017 01:45:17 +0800
Subject: [PATCH 48/59] tx: fix the number of pages is not incorrectly counted

---
 allocate_test.go | 30 ++++++++++++++++++++++++++++++
 tx.go            |  2 +-
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 allocate_test.go

diff --git a/allocate_test.go b/allocate_test.go
new file mode 100644
index 00000000..8566b4df
--- /dev/null
+++ b/allocate_test.go
@@ -0,0 +1,30 @@
+package bolt
+
+import (
+	"testing"
+)
+
+func TestTx_allocatePageStats(t *testing.T) {
+	f := newFreelist()
+	f.ids = []pgid{2, 3}
+
+	tx := &Tx{
+		db: &DB{
+			freelist: f,
+			pageSize: defaultPageSize,
+		},
+		meta:  &meta{},
+		pages: make(map[pgid]*page),
+	}
+
+	prePageCnt := tx.Stats().PageCount
+	allocateCnt := f.free_count()
+
+	if _, err := tx.allocate(allocateCnt); err != nil {
+		t.Fatal(err)
+	}
+
+	if tx.Stats().PageCount != prePageCnt+allocateCnt {
+		t.Errorf("Allocated %d but got %d page in stats", allocateCnt, tx.Stats().PageCount)
+	}
+}
diff --git a/tx.go b/tx.go
index aefcec09..5c029073 100644
--- a/tx.go
+++ b/tx.go
@@ -483,7 +483,7 @@ func (tx *Tx) allocate(count int) (*page, error) {
 	tx.pages[p.id] = p
 
 	// Update statistics.
-	tx.stats.PageCount++
+	tx.stats.PageCount += count
 	tx.stats.PageAlloc += count * tx.db.pageSize
 
 	return p, nil

From 237a4fcb31db8d1249eb47ab773b2dbe2ae8b38b Mon Sep 17 00:00:00 2001
From: Joe Betz <jpbetz@google.com>
Date: Wed, 15 Nov 2017 15:52:34 -0800
Subject: [PATCH 49/59] Panic if page provided to freelist.read is incorrect
 page type.

---
 freelist.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/freelist.go b/freelist.go
index 78e71cbf..318cc3bd 100644
--- a/freelist.go
+++ b/freelist.go
@@ -233,6 +233,9 @@ func (f *freelist) freed(pgid pgid) bool {
 
 // read initializes the freelist from a freelist page.
 func (f *freelist) read(p *page) {
+	if (p.flags & freelistPageFlag) == 0 {
+		panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ()))
+	}
 	// If the page.count is at the max uint16 value (64k) then it's considered
 	// an overflow and the size of the freelist is stored as the first element.
 	idx, count := 0, int(p.count)

From 386b851495d42c4e02908838373a06d0a533e170 Mon Sep 17 00:00:00 2001
From: Anthony Romano <heyitsanthony@users.noreply.github.com>
Date: Thu, 16 Nov 2017 07:57:56 -0800
Subject: [PATCH 50/59] freelist: set alloc tx for freelist to prior txn

Was causing freelist corruption on tx.WriteTo
---
 freelist.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freelist.go b/freelist.go
index 318cc3bd..266f1542 100644
--- a/freelist.go
+++ b/freelist.go
@@ -132,9 +132,9 @@ func (f *freelist) free(txid txid, p *page) {
 	allocTxid, ok := f.allocs[p.id]
 	if ok {
 		delete(f.allocs, p.id)
-	} else if (p.flags & (freelistPageFlag | metaPageFlag)) != 0 {
-		// Safe to claim txid as allocating since these types are private to txid.
-		allocTxid = txid
+	} else if (p.flags & freelistPageFlag) != 0 {
+		// Freelist is always allocated by prior tx.
+		allocTxid = txid - 1
 	}
 
 	for id := p.id; id <= p.id+pgid(p.overflow); id++ {

From 41fefe7322263b61e5669a9bdd136570c15c0c69 Mon Sep 17 00:00:00 2001
From: Anthony Romano <heyitsanthony@users.noreply.github.com>
Date: Thu, 16 Nov 2017 07:58:34 -0800
Subject: [PATCH 51/59] test: check concurrent WriteTo operations aren't
 corrupted

Reliably triggers consistency check failures on ramdisk without freelist
free fix.
---
 db_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/db_test.go b/db_test.go
index 629150c9..e3a58c3c 100644
--- a/db_test.go
+++ b/db_test.go
@@ -9,6 +9,7 @@ import (
 	"hash/fnv"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -588,6 +589,65 @@ func TestDB_BeginRW(t *testing.T) {
 	}
 }
 
+// TestDB_Concurrent_WriteTo checks that issuing WriteTo operations concurrently
+// with commits does not produce corrupted db files.
+func TestDB_Concurrent_WriteTo(t *testing.T) {
+	o := &bolt.Options{NoFreelistSync: false}
+	db := MustOpenWithOption(o)
+	defer db.MustClose()
+
+	var wg sync.WaitGroup
+	wtxs, rtxs := 5, 5
+	wg.Add(wtxs * rtxs)
+	f := func(tx *bolt.Tx) {
+		defer wg.Done()
+		f, err := ioutil.TempFile("", "bolt-")
+		if err != nil {
+			panic(err)
+		}
+		time.Sleep(time.Duration(rand.Intn(20)+1) * time.Millisecond)
+		tx.WriteTo(f)
+		tx.Rollback()
+		f.Close()
+		snap := &DB{nil, f.Name(), o}
+		snap.MustReopen()
+		defer snap.MustClose()
+		snap.MustCheck()
+	}
+
+	tx1, err := db.Begin(true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tx1.CreateBucket([]byte("abc")); err != nil {
+		t.Fatal(err)
+	}
+	if err := tx1.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < wtxs; i++ {
+		tx, err := db.Begin(true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := tx.Bucket([]byte("abc")).Put([]byte{0}, []byte{0}); err != nil {
+			t.Fatal(err)
+		}
+		for j := 0; j < rtxs; j++ {
+			rtx, rerr := db.Begin(false)
+			if rerr != nil {
+				t.Fatal(rerr)
+			}
+			go f(rtx)
+		}
+		if err := tx.Commit(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	wg.Wait()
+}
+
 // Ensure that opening a transaction while the DB is closed returns an error.
 func TestDB_BeginRW_Closed(t *testing.T) {
 	var db bolt.DB

From bcfcdab742efde4bb3b9db28732b9a33c678b523 Mon Sep 17 00:00:00 2001
From: Tommi Virtanen <tv@eagain.net>
Date: Fri, 17 Nov 2017 20:00:15 -0700
Subject: [PATCH 52/59] Remove unnecessary if in batch handling

This is safe, as the only place that creates call values always
explicitly sets err. It's a leftover from an earlier iteration of the
code.
---
 db.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/db.go b/db.go
index 08dfd0ab..4c8c156b 100644
--- a/db.go
+++ b/db.go
@@ -802,9 +802,7 @@ retry:
 
 		// pass success, or bolt internal errors, to all callers
 		for _, c := range b.calls {
-			if c.err != nil {
-				c.err <- err
-			}
+			c.err <- err
 		}
 		break retry
 	}

From cef3333f2a9cc560f5c54f210b8073e9ad273b47 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Thu, 18 Jan 2018 18:01:45 +0100
Subject: [PATCH 53/59] bolt_ppc.go: define `var brokenUnaligned`

If this var is missing building on the ppc architecture fails.
This PR adds it.
---
 bolt_ppc.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bolt_ppc.go b/bolt_ppc.go
index 645ddc3e..55cb8a72 100644
--- a/bolt_ppc.go
+++ b/bolt_ppc.go
@@ -7,3 +7,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

From fafe4b70b5ce332dce497c79d94088b7f2b32404 Mon Sep 17 00:00:00 2001
From: Rodrigo Coelho <rcoelho@users.noreply.github.com>
Date: Tue, 13 Feb 2018 20:02:24 -0200
Subject: [PATCH 54/59] Close waits for the transactions to finish

DB.Close() actually waits for the transactions to finish now, since the PR 377.
https://github.com/boltdb/bolt/pull/377
---
 db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db.go b/db.go
index 4c8c156b..88c13895 100644
--- a/db.go
+++ b/db.go
@@ -441,7 +441,7 @@ func (db *DB) init() error {
 }
 
 // Close releases all database resources.
-// All transactions must be closed before closing the database.
+// It will block waiting for any open transactions to finish before closing the database and returning.
 func (db *DB) Close() error {
 	db.rwlock.Lock()
 	defer db.rwlock.Unlock()

From 584b1a3dba1e7f9aee15721001bf981011e6a232 Mon Sep 17 00:00:00 2001
From: Rodrigo Coelho <rcoelho@users.noreply.github.com>
Date: Wed, 14 Feb 2018 10:06:29 -0200
Subject: [PATCH 55/59] Breaking the long line

---
 db.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/db.go b/db.go
index 88c13895..d5c53f4a 100644
--- a/db.go
+++ b/db.go
@@ -441,7 +441,8 @@ func (db *DB) init() error {
 }
 
 // Close releases all database resources.
-// It will block waiting for any open transactions to finish before closing the database and returning.
+// It will block waiting for any open transactions to finish
+// before closing the database and returning.
 func (db *DB) Close() error {
 	db.rwlock.Lock()
 	defer db.rwlock.Unlock()

From eb69cfa87f4da99712e2a9d30aa8bd83e10c68d7 Mon Sep 17 00:00:00 2001
From: lukechampine <luke.champine@gmail.com>
Date: Wed, 14 Feb 2018 17:40:35 -0500
Subject: [PATCH 56/59] change import to NebulousLabs/bolt

---
 Makefile                            | 2 +-
 bucket_test.go                      | 2 +-
 cmd/bolt/main.go                    | 2 +-
 cmd/bolt/main_test.go               | 4 ++--
 cursor_test.go                      | 2 +-
 db_test.go                          | 2 +-
 simulation_no_freelist_sync_test.go | 2 +-
 simulation_test.go                  | 2 +-
 tx_test.go                          | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 43b94f3b..9dedcc36 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ unused:
 
 # go get github.com/kisielk/errcheck
 errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt
+	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/NebulousLabs/bolt
 
 test:
 	go test -timeout 20m -v -coverprofile cover.out -covermode atomic
diff --git a/bucket_test.go b/bucket_test.go
index b7ce32c1..7ae670b7 100644
--- a/bucket_test.go
+++ b/bucket_test.go
@@ -13,7 +13,7 @@ import (
 	"testing"
 	"testing/quick"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 // Ensure that a bucket that gets a non-existent key returns nil.
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index eb85e05c..a2969fc7 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -19,7 +19,7 @@ import (
 	"unicode/utf8"
 	"unsafe"
 
-	bolt "github.com/coreos/bbolt"
+	bolt "github.com/NebulousLabs/bolt"
 )
 
 var (
diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go
index 16bf804a..9173ff09 100644
--- a/cmd/bolt/main_test.go
+++ b/cmd/bolt/main_test.go
@@ -12,8 +12,8 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/coreos/bbolt"
-	"github.com/coreos/bbolt/cmd/bolt"
+	"github.com/NebulousLabs/bolt"
+	"github.com/NebulousLabs/bolt/cmd/bolt"
 )
 
 // Ensure the "info" command can print information about a database.
diff --git a/cursor_test.go b/cursor_test.go
index 7b1ae198..953cca36 100644
--- a/cursor_test.go
+++ b/cursor_test.go
@@ -11,7 +11,7 @@ import (
 	"testing"
 	"testing/quick"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 // Ensure that a cursor can return a reference to the bucket that created it.
diff --git a/db_test.go b/db_test.go
index e3a58c3c..6035ab1f 100644
--- a/db_test.go
+++ b/db_test.go
@@ -18,7 +18,7 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 var statsFlag = flag.Bool("stats", false, "show performance stats")
diff --git a/simulation_no_freelist_sync_test.go b/simulation_no_freelist_sync_test.go
index da2031ee..21ec22d2 100644
--- a/simulation_no_freelist_sync_test.go
+++ b/simulation_no_freelist_sync_test.go
@@ -3,7 +3,7 @@ package bolt_test
 import (
 	"testing"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 func TestSimulateNoFreeListSync_1op_1p(t *testing.T) {
diff --git a/simulation_test.go b/simulation_test.go
index a5889c02..633fc9f7 100644
--- a/simulation_test.go
+++ b/simulation_test.go
@@ -7,7 +7,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 func TestSimulate_1op_1p(t *testing.T)     { testSimulate(t, nil, 1, 1, 1) }
diff --git a/tx_test.go b/tx_test.go
index de92cb53..83aabe3f 100644
--- a/tx_test.go
+++ b/tx_test.go
@@ -8,7 +8,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/coreos/bbolt"
+	"github.com/NebulousLabs/bolt"
 )
 
 // TestTx_Check_ReadOnly tests consistency checking on a ReadOnly database.

From 86a90cdb2147f8bf0de6e05f98a64a6bf3c3a340 Mon Sep 17 00:00:00 2001
From: lukechampine <luke.champine@gmail.com>
Date: Tue, 18 Jul 2017 16:06:27 -0400
Subject: [PATCH 57/59] remap old size if new mmap fails

---
 db.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/db.go b/db.go
index d5c53f4a..44579e37 100644
--- a/db.go
+++ b/db.go
@@ -332,6 +332,11 @@ func (db *DB) mmap(minsz int) error {
 
 	// Memory-map the data file as a byte slice.
 	if err := mmap(db, size); err != nil {
+		// mmap failed; the system may have run out of space. Fallback to
+		// mapping the bare minimum needed for the current db size.
+		if err2 := mmap(db, db.datasz); err2 != nil {
+			panic(fmt.Sprintf("failed to revert db size after failed mmap: %v", err2))
+		}
 		return err
 	}
 

From 97348af0a12d1e5ce5b3e0741f0091c7d36d96f2 Mon Sep 17 00:00:00 2001
From: lukechampine <luke.champine@gmail.com>
Date: Tue, 18 Jul 2017 16:11:56 -0400
Subject: [PATCH 58/59] add MmapError type

---
 db.go     | 2 +-
 errors.go | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/db.go b/db.go
index 44579e37..ec040eeb 100644
--- a/db.go
+++ b/db.go
@@ -337,7 +337,7 @@ func (db *DB) mmap(minsz int) error {
 		if err2 := mmap(db, db.datasz); err2 != nil {
 			panic(fmt.Sprintf("failed to revert db size after failed mmap: %v", err2))
 		}
-		return err
+		return MmapError(err)
 	}
 
 	// Save references to the meta pages.
diff --git a/errors.go b/errors.go
index a3620a3e..afa1e587 100644
--- a/errors.go
+++ b/errors.go
@@ -69,3 +69,8 @@ var (
 	// non-bucket key on an existing bucket key.
 	ErrIncompatibleValue = errors.New("incompatible value")
 )
+
+// MmapError represents an error resulting from a failed mmap call. Typically,
+// this error means that no further database writes will be possible. The most
+// common cause is insufficient disk space.
+type MmapError error

From 4917b25505fa26c2516a1c77dbc3fd2a29bfcfc4 Mon Sep 17 00:00:00 2001
From: lukechampine <luke.champine@gmail.com>
Date: Thu, 20 Jul 2017 00:54:08 -0400
Subject: [PATCH 59/59] fix mmap error type

---
 db.go     | 2 +-
 errors.go | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/db.go b/db.go
index ec040eeb..54240c52 100644
--- a/db.go
+++ b/db.go
@@ -337,7 +337,7 @@ func (db *DB) mmap(minsz int) error {
 		if err2 := mmap(db, db.datasz); err2 != nil {
 			panic(fmt.Sprintf("failed to revert db size after failed mmap: %v", err2))
 		}
-		return MmapError(err)
+		return MmapError(err.Error())
 	}
 
 	// Save references to the meta pages.
diff --git a/errors.go b/errors.go
index afa1e587..d0e4eafa 100644
--- a/errors.go
+++ b/errors.go
@@ -73,4 +73,6 @@ var (
 // MmapError represents an error resulting from a failed mmap call. Typically,
 // this error means that no further database writes will be possible. The most
 // common cause is insufficient disk space.
-type MmapError error
+type MmapError string
+
+func (e MmapError) Error() string { return string(e) }