Merge pull request #857 from hashicorp/f-boltdb

Raft uses BoltDB

command/agent/command.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
+	"github.com/hashicorp/consul-migrate/migrator"
 	"github.com/hashicorp/consul/watch"
 	"github.com/hashicorp/go-checkpoint"
 	"github.com/hashicorp/go-syslog"
@@ -598,6 +599,28 @@ func (c *Command) Run(args []string) int {
 		metrics.NewGlobal(metricsConf, inm)
 	}
 
+	// If we are starting a consul 0.5.1+ server for the first time,
+	// and we have data from a previous Consul version, attempt to
+	// migrate the data from LMDB to BoltDB using the migrator utility.
+	if config.Server {
+		m, err := migrator.New(config.DataDir)
+		if err != nil {
+			c.Ui.Error(err.Error())
+			return 1
+		}
+
+		start := time.Now()
+		migrated, err := m.Migrate()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Failed to migrate raft data: %s", err))
+			return 1
+		}
+		if migrated {
+			duration := time.Now().Sub(start)
+			c.Ui.Output(fmt.Sprintf("Successfully migrated raft data in %s", duration))
+		}
+	}
+
 	// Create the agent
 	if err := c.setupAgent(config, logOutput, logWriter); err != nil {
 		return 1

consul/server.go

@@ -10,15 +10,14 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/golang-lru"
 	"github.com/hashicorp/raft"
-	"github.com/hashicorp/raft-mdb"
+	"github.com/hashicorp/raft-boltdb"
 	"github.com/hashicorp/serf/serf"
 )
 
@@ -31,13 +30,11 @@ const (
 )
 
 const (
-	serfLANSnapshot          = "serf/local.snapshot"
-	serfWANSnapshot          = "serf/remote.snapshot"
-	raftState                = "raft/"
-	tmpStatePath             = "tmp/"
-	snapshotsRetained        = 2
-	raftDBSize32bit   uint64 = 64 * 1024 * 1024       // Limit Raft log to 64MB
-	raftDBSize64bit   uint64 = 8 * 1024 * 1024 * 1024 // Limit Raft log to 8GB
+	serfLANSnapshot   = "serf/local.snapshot"
+	serfWANSnapshot   = "serf/remote.snapshot"
+	raftState         = "raft/"
+	tmpStatePath      = "tmp/"
+	snapshotsRetained = 2
 
 	// serverRPCCache controls how long we keep an idle connection
 	// open to a server
@@ -108,7 +105,7 @@ type Server struct {
 	raft          *raft.Raft
 	raftLayer     *RaftLayer
 	raftPeers     raft.PeerStore
-	raftStore     *raftmdb.MDBStore
+	raftStore     *raftboltdb.BoltStore
 	raftTransport *raft.NetworkTransport
 
 	// reconcileCh is used to pass events from the serf handler
@@ -349,22 +346,14 @@ func (s *Server) setupRaft() error {
 		return err
 	}
 
-	// Set the maximum raft size based on 32/64bit. Since we are
-	// doing an mmap underneath, we need to limit our use of virtual
-	// address space on 32bit, but don't have to care on 64bit.
-	dbSize := raftDBSize32bit
-	if runtime.GOARCH == "amd64" {
-		dbSize = raftDBSize64bit
-	}
-
 	// Create the base raft path
 	path := filepath.Join(s.config.DataDir, raftState)
 	if err := ensurePath(path, true); err != nil {
 		return err
 	}
 
-	// Create the MDB store for logs and stable storage
-	store, err := raftmdb.NewMDBStoreWithSize(path, dbSize)
+	// Create the backend raft store for logs and stable storage
+	store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db"))
 	if err != nil {
 		return err
 	}

website/source/docs/upgrade-specific.html.markdown

@@ -14,6 +14,28 @@ details provided for their upgrades as a result of new features or changed
 behavior. This page is used to document those details seperately from the
 standard upgrade flow.
 
+## Consul 0.5.1
+
+Consul version 0.5.1 uses a different backend store for persisting the Raft
+log. Because of this change, a data migration is necessary to move the log
+entries out of LMDB and into the newer backend, BoltDB.
+
+Consul version 0.5.1 makes this transition seamless and easy. As a user, there
+are no special steps you need to take. When Consul 0.5.1 starts, it checks
+for presence of the legacy LMDB data files, and migrates them automatically
+if any are found. You will see a log emitted when Raft data is migrated, like
+this:
+
+```
+==> Successfully migrated raft data in 5.839642ms
+```
+
+The automatic upgrade will only exist in Consul 0.5.1. In later versions
+(0.6.0+), the migration code will not be included in the Consul binary. It
+is still possible to upgrade directly from pre-0.5.1 versions by using the
+consul-migrate utility, which is available on the
+[Consul Tools page](/downloads_tools.html).
+
 ## Consul 0.5
 
 Consul version 0.5 adds two features that complicate the upgrade process:

website/source/downloads_tools.html.erb

@@ -29,6 +29,9 @@ description: |-
       <li>
         <a href="https://github.com/hashicorp/consul-template">Consul Template</a> - Generic template rendering and notifications with Consul
       </li>
+      <li>
+        <a href="https://github.com/hashicorp/consul-migrate">Consul Migrate</a> - Data migration tool to handle Consul upgrades to 0.5.1+
+      </li>
     </ul>
   </div>