From d901fa638970cf0f2efe2a547fd867dc2c639d5b Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Thu, 15 Oct 2015 14:14:56 -0700 Subject: [PATCH 1/5] agent: remove migrator, refuse to start if mdb dir found --- command/agent/command.go | 90 ++++++++++------------------------------ 1 file changed, 23 insertions(+), 67 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index 5530b7e0fede..f7b5dbf3d97a 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -15,7 +15,6 @@ import ( "time" "github.com/armon/go-metrics" - "github.com/hashicorp/consul-migrate/migrator" "github.com/hashicorp/consul/watch" "github.com/hashicorp/go-checkpoint" "github.com/hashicorp/go-syslog" @@ -165,6 +164,18 @@ func (c *Command) readConfig() *Config { return nil } + // Check the data dir for signs of an un-migrated Consul 0.5.x or older + // server. Consul refuses to start if this is present to protect a server + // with existing data from starting on a fresh data set. + if config.Server { + if err := checkDataFormat(config.DataDir); err != nil { + c.Ui.Error(fmt.Sprintf("CRITICAL: %v", err)) + c.Ui.Error("Consul will refuse to boot with this directory present.") + c.Ui.Error("See https://consul.io/docs/upgrade-specific.html for more information.") + return nil + } + } + if config.EncryptKey != "" { if _, err := config.EncryptBytes(); err != nil { c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err)) @@ -601,72 +612,6 @@ func (c *Command) Run(args []string) int { metrics.NewGlobal(metricsConf, inm) } - // If we are starting a consul 0.5.1+ server for the first time, - // and we have data from a previous Consul version, attempt to - // migrate the data from LMDB to BoltDB using the migrator utility. - if config.Server { - // If the data dir doesn't exist yet (first start), then don't - // attempt to migrate. - if _, err := os.Stat(config.DataDir); os.IsNotExist(err) { - goto AFTER_MIGRATE - } - - m, err := migrator.New(config.DataDir) - if err != nil { - c.Ui.Error(err.Error()) - return 1 - } - - // Handle progress info from the migrator utility. This will - // just dump out the current operation and progress every ~5 - // percent progress. - doneCh := make(chan struct{}) - go func() { - var lastOp string - var lastProgress float64 - lastFlush := time.Now() - for { - select { - case update := <-m.ProgressCh: - switch { - case lastOp != update.Op: - lastProgress = update.Progress - lastOp = update.Op - c.Ui.Output(update.Op) - c.Ui.Info(fmt.Sprintf("%.2f%%", update.Progress)) - - case update.Progress-lastProgress >= 5: - fallthrough - - case time.Now().Sub(lastFlush) > time.Second: - fallthrough - - case update.Progress == 100: - lastFlush = time.Now() - lastProgress = update.Progress - c.Ui.Info(fmt.Sprintf("%.2f%%", update.Progress)) - } - case <-doneCh: - return - } - } - }() - - c.Ui.Output("Starting raft data migration...") - start := time.Now() - migrated, err := m.Migrate() - close(doneCh) - if err != nil { - c.Ui.Error(fmt.Sprintf("Failed to migrate raft data: %s", err)) - return 1 - } - if migrated { - duration := time.Now().Sub(start) - c.Ui.Output(fmt.Sprintf("Successfully migrated raft data in %s", duration)) - } - } - -AFTER_MIGRATE: // Create the agent if err := c.setupAgent(config, logOutput, logWriter); err != nil { return 1 @@ -921,6 +866,17 @@ func (c *Command) handleReload(config *Config) *Config { return newConf } +// checkDataFormat checks the Consul data directory for the deprecated "mdb" +// folder. If it exists, Consul will fail to boot, avoiding situations where +// non-migrated servers start with a fresh data set. +func checkDataFormat(dataDir string) error { + mdbPath := filepath.Join(dataDir, "mdb") + if _, err := os.Stat(mdbPath); !os.IsNotExist(err) { + return fmt.Errorf("Deprecated data folder found at %q!", mdbPath) + } + return nil +} + // startScadaClient is used to start a new SCADA provider and listener, // replacing any existing listeners. func (c *Command) setupScadaConn(config *Config) error { From 10b971df2124f9891ece01abeada28b2c8d7f667 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Thu, 15 Oct 2015 14:15:41 -0700 Subject: [PATCH 2/5] agent: test mdb dir protection --- command/agent/command_test.go | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/command/agent/command_test.go b/command/agent/command_test.go index 9839446981bd..9f9317e7bd0e 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "log" "os" + "path/filepath" "strings" "testing" @@ -299,3 +300,39 @@ func TestSetupScadaConn(t *testing.T) { t.Fatalf("should be closed") } } + +func TestProtectDataDir(t *testing.T) { + dir, err := ioutil.TempDir("", "consul") + if err != nil { + t.Fatalf("err: %v", err) + } + defer os.RemoveAll(dir) + + if err := os.MkdirAll(filepath.Join(dir, "mdb"), 0700); err != nil { + t.Fatalf("err: %v", err) + } + + cfgFile, err := ioutil.TempFile("", "consul") + if err != nil { + t.Fatalf("err: %v", err) + } + defer os.Remove(cfgFile.Name()) + + content := fmt.Sprintf(`{"server": true, "data_dir": "%s"}`, dir) + _, err = cfgFile.Write([]byte(content)) + if err != nil { + t.Fatalf("err: %v", err) + } + + ui := new(cli.MockUi) + cmd := &Command{ + Ui: ui, + args: []string{"-config-file=" + cfgFile.Name()}, + } + if conf := cmd.readConfig(); conf != nil { + t.Fatalf("should fail") + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, dir) { + t.Fatalf("expected mdb dir error, got: %s", out) + } +} From de287e3efb961112ab359de63eea650ff3b4e8e2 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Thu, 15 Oct 2015 14:21:35 -0700 Subject: [PATCH 3/5] agent: consolidates data dir checker --- command/agent/command.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index f7b5dbf3d97a..2d642b20e310 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -168,8 +168,9 @@ func (c *Command) readConfig() *Config { // server. Consul refuses to start if this is present to protect a server // with existing data from starting on a fresh data set. if config.Server { - if err := checkDataFormat(config.DataDir); err != nil { - c.Ui.Error(fmt.Sprintf("CRITICAL: %v", err)) + mdbPath := filepath.Join(config.DataDir, "mdb") + if _, err := os.Stat(mdbPath); !os.IsNotExist(err) { + c.Ui.Error(fmt.Sprintf("CRITICAL: Deprecated data folder found at %q!", mdbPath)) c.Ui.Error("Consul will refuse to boot with this directory present.") c.Ui.Error("See https://consul.io/docs/upgrade-specific.html for more information.") return nil @@ -866,17 +867,6 @@ func (c *Command) handleReload(config *Config) *Config { return newConf } -// checkDataFormat checks the Consul data directory for the deprecated "mdb" -// folder. If it exists, Consul will fail to boot, avoiding situations where -// non-migrated servers start with a fresh data set. -func checkDataFormat(dataDir string) error { - mdbPath := filepath.Join(dataDir, "mdb") - if _, err := os.Stat(mdbPath); !os.IsNotExist(err) { - return fmt.Errorf("Deprecated data folder found at %q!", mdbPath) - } - return nil -} - // startScadaClient is used to start a new SCADA provider and listener, // replacing any existing listeners. func (c *Command) setupScadaConn(config *Config) error { From 3872e57b3bfba4e62b53d2b8fdfbf620055078eb Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Thu, 15 Oct 2015 14:42:46 -0700 Subject: [PATCH 4/5] website: add note for consul migrate utility. --- .../docs/upgrade-specific.html.markdown | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/website/source/docs/upgrade-specific.html.markdown b/website/source/docs/upgrade-specific.html.markdown index 1841d68df9a5..6f761f43c49b 100644 --- a/website/source/docs/upgrade-specific.html.markdown +++ b/website/source/docs/upgrade-specific.html.markdown @@ -16,6 +16,26 @@ standard upgrade flow. ## Consul 0.6 +Consul version 0.6 is a very large release with many enhancements and +optimizations. Changes to be aware of during an upgrade are categorized below. + +#### Data store changes + +Consul changed the format used to store data on the server nodes in version 0.5 +(see 0.5.1 notes below for details). Previously, Consul would automatically +detect data directories using the old LMDB format, and convert them to the newer +BoltDB format. This automatic upgrade has been removed for Consul 0.6, and +instead a safeguard has been put in place which will prevent Consul from booting +if the old directory format is detected. + +It is still possible to migrate from a 0.5.x version of Consul to 0.6+ using the +[consul-migrate](https://github.com/hashicorp/consul-migrate) CLI utility. This +is the same tool that was previously embedded into Consul. See the +[releases](https://github.com/hashicorp/consul-migrate/releases) page for +downloadable versions of the tool. + +#### ACL Enhancements + Consul 0.6 introduces enhancements to the ACL system which may require special handling: From bdd1a9d064317d5ae86e7eec38fe8044db88bb26 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Thu, 15 Oct 2015 14:48:59 -0700 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e9873371d56..bd00bed0e051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ IMPROVEMENTS: MISC: * Vagrantfile fixed for VMware [GH-1042] +* Data migrator utility removed to reduce cgo dependency. [GH-1309] ## 0.5.2 (May 18, 2015) @@ -36,6 +37,13 @@ MISC: * Remove unused constant [GH-941] +UPGRADE NOTES: + +* Consul will refuse to start if the data directory contains an "mdb" folder. + This folder was used in versions of Consul up to 0.5.1. Consul version 0.5.2 + included a baked-in utility to automatically upgrade the data format, but + this has been removed in Consul 0.6 to reduce the dependency on cgo. + ## 0.5.1 (May 13, 2015) FEATURES: