Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(upgrade): create CLI configs #19937

Merged
merged 6 commits into from
Nov 9, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
1. [19924](https://github.com/influxdata/influxdb/pull/19924): Remove unused 'security-script' option from upgrade command
1. [19928](https://github.com/influxdata/influxdb/pull/19928): Fix parsing of retention policy CLI args in `influx setup` and `influxd upgrade`
1. [19952](https://github.com/influxdata/influxdb/pull/19952): Use `db`/`rp` naming convention when migrating DBs to buckets
1. [19925](https://github.com/influxdata/influxdb/pull/19937): Create CLI configs in `influxd upgrade`

## v2.0.0-rc.4 [2020-11-05]

Expand Down
11 changes: 7 additions & 4 deletions cmd/influxd/upgrade/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,9 @@ func TestConfigUpgrade(t *testing.T) {
}

var typicalRetval, emptyRetval configV1
_, err := toml.Decode("[meta]\ndir=\"/var/lib/influxdb/meta\"\n[data]\ndir=\"/var/lib/influxdb/data\"\nwal-dir=\"/var/lib/influxdb/wal\"\n",
_, err := toml.Decode("[meta]\ndir=\"/var/lib/influxdb/meta\"\n[data]\ndir=\"/var/lib/influxdb/data\"\nwal-dir=\"/var/lib/influxdb/wal\"\n[http]\nbind-address=\":8086\"\nhttps-enabled=false",
&typicalRetval)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)

type testCase struct {
name string
Expand Down Expand Up @@ -323,6 +321,10 @@ reporting-disabled = true
dir = "/var/lib/influxdb/data"
wal-dir = "/var/lib/influxdb/wal"

[http]
enabled = true
bind-address = ":8086"

[[udp]]
enabled = false
bind-address = ":8089"
Expand Down Expand Up @@ -395,6 +397,7 @@ tls-key = ""
var testConfigV2obsoleteArrays = `reporting-disabled = true
bolt-path = "/db/.influxdbv2/influxd.bolt"
engine-path = "/db/.influxdbv2/engine"
http-bind-address = ":8086"
`

var testConfigV2empty = `
Expand Down
27 changes: 26 additions & 1 deletion cmd/influxd/upgrade/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@ package upgrade

import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"time"

"github.com/influxdata/influxdb/v2"
"github.com/influxdata/influxdb/v2/cmd/influx/config"
"github.com/influxdata/influxdb/v2/cmd/internal"
"github.com/tcnksm/go-input"
"go.uber.org/zap"
)

func setupAdmin(ctx context.Context, v2 *influxDBv2, req *influxdb.OnboardingRequest) (*influxdb.OnboardingResults, error) {
Expand Down Expand Up @@ -123,9 +127,30 @@ You have entered:
Retention Period: %s
`, req.User, req.Org, req.Bucket, rp)
}); !confirmed {
return nil, fmt.Errorf("setup was canceled")
return nil, errors.New("setup was canceled")
}
}

return req, nil
}

func saveLocalConfig(sourceOptions *optionsV1, targetOptions *optionsV2, log *zap.Logger) error {
dPath, dir := targetOptions.configsPath, filepath.Dir(targetOptions.configsPath)
if dPath == "" || dir == "" {
return errors.New("a valid configurations path must be provided")
}
localConfigSVC := config.NewLocalConfigSVC(dPath, dir)
p := config.DefaultConfig
p.Token = targetOptions.token
p.Org = targetOptions.orgName
if sourceOptions.dbURL != "" {
p.Host = sourceOptions.dbURL
}
if _, err := localConfigSVC.CreateConfig(p); err != nil {
log.Error("failed to save CLI config", zap.String("path", dPath), zap.Error(err))
return errors.New("failed to save CLI config")
}
log.Info("CLI config has been stored.", zap.String("path", dPath))

return nil
}
83 changes: 83 additions & 0 deletions cmd/influxd/upgrade/setup_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package upgrade

import (
"path/filepath"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/influxdata/influxdb/v2/bolt"
"github.com/influxdata/influxdb/v2/cmd/influx/config"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)

func TestLocalConfig(t *testing.T) {

type testCase struct {
name string
sourceOpts *optionsV1
targetOpts *optionsV2
want config.Config
}

var testCases = []testCase{
{
name: "default",
sourceOpts: &optionsV1{},
targetOpts: &optionsV2{
orgName: "my-org",
token: "my-token",
},
want: config.Config{
Name: config.DefaultConfig.Name,
Host: config.DefaultConfig.Host,
Org: "my-org",
Token: "my-token",
Active: config.DefaultConfig.Active,
},
},
{
name: "v1 url",
sourceOpts: &optionsV1{
dbURL: "https://10.0.0.1:8086",
},
targetOpts: &optionsV2{
orgName: "my-org",
token: "my-token",
},
want: config.Config{
Name: config.DefaultConfig.Name,
Host: "https://10.0.0.1:8086",
Org: "my-org",
Token: "my-token",
Active: config.DefaultConfig.Active,
},
},
}

for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
log := zaptest.NewLogger(t)
// adjust paths to runtime values
opts := tc.targetOpts
opts.boltPath = filepath.Join(dir, bolt.DefaultFilename)
opts.configsPath = filepath.Join(dir, "configs")
opts.enginePath = filepath.Join(dir, "engine")
// execute op
err := saveLocalConfig(tc.sourceOpts, opts, log)
require.NoError(t, err)
// verify saved config
dPath, dir := opts.configsPath, filepath.Dir(opts.configsPath)
localConfigSVC := config.NewLocalConfigSVC(dPath, dir)
cs, err := localConfigSVC.ListConfigs()
require.NoError(t, err)
actual := cs.Active()
if diff := cmp.Diff(tc.want, actual); diff != "" {
t.Fatal(diff)
}
})
}
}
41 changes: 41 additions & 0 deletions cmd/influxd/upgrade/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import (
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"strings"

"github.com/influxdata/influxdb/v2"
"github.com/influxdata/influxdb/v2/bolt"
Expand Down Expand Up @@ -40,12 +42,36 @@ type configV1 struct {
Dir string `toml:"dir"`
WALDir string `toml:"wal-dir"`
} `toml:"data"`
Http struct {
docmerlin marked this conversation as resolved.
Show resolved Hide resolved
BindAddress string `toml:"bind-address"`
HttpsEnabled bool `toml:"https-enabled"`
} `toml:"http"`
}

func (c *configV1) dbURL() string {
address := c.Http.BindAddress
if address == "" { // fallback to default
address = ":8086"
}
var url url.URL
if c.Http.HttpsEnabled {
url.Scheme = "https"
} else {
url.Scheme = "http"
}
if strings.HasPrefix(address, ":") { // address is just :port
url.Host = "localhost" + address
} else {
url.Host = address
}
return url.String()
}

type optionsV1 struct {
metaDir string
walDir string
dataDir string
dbURL string
// cmd option
dbDir string
configFile string
Expand All @@ -66,6 +92,7 @@ func (o *optionsV1) checkDirs() error {

type optionsV2 struct {
boltPath string
configsPath string
enginePath string
userName string
password string
Expand Down Expand Up @@ -121,6 +148,7 @@ func NewCommand() *cobra.Command {
Upgrades a 1.x version of InfluxDB by performing the following actions:
1. Reads the 1.x config file and creates a 2.x config file with matching options. Unsupported 1.x options are reported.
2. Copies 1.x database files.
3. Creates influx CLI configurations.

If the config file is not available, 1.x db folder (--v1-dir options) is taken as an input.
Target 2.x database dir is specified by the --engine-path option. If changed, the bolt path should be changed as well.
Expand Down Expand Up @@ -149,6 +177,13 @@ func NewCommand() *cobra.Command {
Desc: "path for boltdb database",
alespour marked this conversation as resolved.
Show resolved Hide resolved
Short: 'm',
},
{
DestP: &options.target.configsPath,
Flag: "influx-configs-path",
Default: filepath.Join(v2dir, "configs"),
Desc: "path for CLI configurations",
Short: 'c',
},
{
DestP: &options.target.enginePath,
Flag: "engine-path",
Expand Down Expand Up @@ -307,6 +342,7 @@ func runUpgradeE(*cobra.Command, []string) error {
options.source.metaDir = v1Config.Meta.Dir
options.source.dataDir = v1Config.Data.Dir
options.source.walDir = v1Config.Data.WALDir
options.source.dbURL = v1Config.dbURL()
} else {
log.Info("No InfluxDB 1.x config file specified, skipping its upgrade")
}
Expand Down Expand Up @@ -374,6 +410,11 @@ func runUpgradeE(*cobra.Command, []string) error {
options.target.userID = or.User.ID
options.target.token = or.Auth.Token

err = saveLocalConfig(&options.source, &options.target, log)
if err != nil {
return err
}

db2BucketIds, err := upgradeDatabases(ctx, v1, v2, &options.source, &options.target, or.Org.ID, log)
if err != nil {
//remove all files
Expand Down
49 changes: 49 additions & 0 deletions cmd/influxd/upgrade/upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"path/filepath"
"testing"

"github.com/BurntSushi/toml"
"github.com/google/go-cmp/cmp"
"github.com/influxdata/influxdb/v2/bolt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -21,6 +23,7 @@ func TestPathValidations(t *testing.T) {
v2Dir := filepath.Join(tmpdir, "v2db")

boltPath := filepath.Join(v2Dir, bolt.DefaultFilename)
configsPath := filepath.Join(v2Dir, "configs")
enginePath := filepath.Join(v2Dir, "engine")

err = os.MkdirAll(filepath.Join(enginePath, "db"), 0777)
Expand All @@ -35,6 +38,7 @@ func TestPathValidations(t *testing.T) {
largs = append(largs, "--token", "my-token")
largs = append(largs, "--v1-dir", v1Dir)
largs = append(largs, "--bolt-path", boltPath)
largs = append(largs, "--influx-configs-path", configsPath)
largs = append(largs, "--engine-path", enginePath)
largs = append(largs, "--config-file", "")

Expand Down Expand Up @@ -67,3 +71,48 @@ func TestPathValidations(t *testing.T) {
require.NotNil(t, err, "Must fail")
assert.Contains(t, err.Error(), "target engine path")
}

func TestDbURL(t *testing.T) {

type testCase struct {
name string
conf string
want string
}

var testCases = []testCase{
{
name: "default",
conf: "[meta]\n[data]\n[http]\n",
want: "http://localhost:8086",
},
{
name: "custom but same as default",
conf: "[meta]\n[data]\n[http]\nbind-address=\":8086\"\nhttps-enabled=false",
want: "http://localhost:8086",
},
{
name: "custom no host",
conf: "[meta]\n[data]\n[http]\nbind-address=\":8186\"\nhttps-enabled=true",
want: "https://localhost:8186",
},
{
name: "custom with host",
conf: "[meta]\n[data]\n[http]\nbind-address=\"10.0.0.1:8086\"\nhttps-enabled=true",
want: "https://10.0.0.1:8086",
},
}

for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var c configV1
_, err := toml.Decode(tc.conf, &c)
require.NoError(t, err)
if diff := cmp.Diff(tc.want, c.dbURL()); diff != "" {
t.Fatal(diff)
}
})
}
}