Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Require specifying ports when using multiple nodes #145

Merged
merged 2 commits into from
Sep 8, 2022
Merged

Require specifying ports when using multiple nodes #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ed81c0
require specifying ports when using multiple nodes
RichardJCai Sep 6, 2022
1da40a5
allow specifying http ports for multi-node setup
RichardJCai Sep 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 52 additions & 7 deletions testserver/testserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,8 @@ type testServerArgs struct {
rootPW string // if nonempty, set as pw for root
storeOnDisk bool // to save database in disk
storeMemSize float64 // the proportion of available memory allocated to test server
httpPort int
httpPorts []int
listenAddrPorts []int
testConfig TestConfig
nonStableDB bool
cockroachBinary string // path to cockroach executable file
Expand Down Expand Up @@ -287,9 +288,29 @@ func NonStableDbOpt() TestServerOpt {

// ExposeConsoleOpt is a TestServer option that can be passed to NewTestServer to
// expose the console of the server on the given port.
// Warning: This is kept around for backwards compatibility, use AddHttpPortOpt
// instead.
func ExposeConsoleOpt(port int) TestServerOpt {
return func(args *testServerArgs) {
args.httpPort = port
args.httpPorts = []int{port}
}
}

// AddHttpPortOpt is a TestServer option that can be passed to NewTestServer to
// specify the http ports for the Cockroach nodes.
func AddHttpPortOpt(port int) TestServerOpt {
return func(args *testServerArgs) {
args.httpPorts = append(args.httpPorts, port)
}
}

// AddListenAddrPortOpt is a TestServer option that can be passed to NewTestServer to
// specify the ports for the Cockroach nodes.
// In the case of restarting nodes, it is up to the user of TestServer to make
// sure the port used here cannot be re-used.
func AddListenAddrPortOpt(port int) TestServerOpt {
return func(args *testServerArgs) {
args.listenAddrPorts = append(args.listenAddrPorts, port)
}
}

Expand Down Expand Up @@ -337,6 +358,19 @@ func NewTestServer(opts ...TestServerOpt) (TestServer, error) {
serverArgs.cockroachBinary = customBinaryEnv
}

// For backwards compatibility, in the 3 node case where no args are
// specified, default to ports 26257, 26258, 26259.
if serverArgs.numNodes == 3 && len(serverArgs.listenAddrPorts) == 0 {
serverArgs.listenAddrPorts = []int{26257, 26258, 26259}
} else if serverArgs.numNodes != 1 && len(serverArgs.listenAddrPorts) != serverArgs.numNodes {
panic(fmt.Sprintf("need to specify a port for each node using AddListenAddrPortOpt, got %d nodes, need %d ports",
serverArgs.numNodes, len(serverArgs.listenAddrPorts)))
}

if len(serverArgs.listenAddrPorts) == 0 || len(serverArgs.listenAddrPorts) == 1 {
serverArgs.listenAddrPorts = []int{0}
}

var err error
if serverArgs.cockroachBinary != "" {
log.Printf("Using custom cockroach binary: %s", serverArgs.cockroachBinary)
Expand Down Expand Up @@ -445,19 +479,30 @@ func NewTestServer(opts ...TestServerOpt) (TestServer, error) {

nodes := make([]nodeInfo, serverArgs.numNodes)
var initArgs []string
joinAddrs := make([]string, 3)
hostPort := serverArgs.listenAddrPorts[0]
for i, port := range serverArgs.listenAddrPorts {
joinAddrs[i] = fmt.Sprintf("localhost:%d", port)
}

if len(serverArgs.httpPorts) == 0 {
serverArgs.httpPorts = make([]int, serverArgs.numNodes)
}

for i := 0; i < serverArgs.numNodes; i++ {
nodes[i].state = stateNew
nodes[i].listeningURLFile = filepath.Join(baseDir, fmt.Sprintf("listen-url%d", i))
if serverArgs.numNodes > 1 {
joinArg := fmt.Sprintf("--join=%s", strings.Join(joinAddrs, ","))
nodes[i].startCmdArgs = []string{
serverArgs.cockroachBinary,
startCmd,
secureOpt,
storeArg + strconv.Itoa(i),
fmt.Sprintf("--listen-addr=localhost:%d", 26257+i),
fmt.Sprintf("--http-addr=localhost:%d", 8080+i),
fmt.Sprintf("--listen-addr=localhost:%d", serverArgs.listenAddrPorts[i]),
fmt.Sprintf("--http-addr=localhost:%d", serverArgs.httpPorts[i]),
"--listening-url-file=" + nodes[i].listeningURLFile,
fmt.Sprintf("--join=localhost:%d,localhost:%d,localhost:%d", 26257, 26258, 26259),
joinArg,
}
} else {
nodes[0].startCmdArgs = []string{
Expand All @@ -467,7 +512,7 @@ func NewTestServer(opts ...TestServerOpt) (TestServer, error) {
secureOpt,
"--host=localhost",
"--port=0",
"--http-port=" + strconv.Itoa(serverArgs.httpPort),
"--http-port=" + strconv.Itoa(serverArgs.httpPorts[0]),
storeArg,
"--listening-url-file=" + nodes[i].listeningURLFile,
}
Expand All @@ -480,7 +525,7 @@ func NewTestServer(opts ...TestServerOpt) (TestServer, error) {
serverArgs.cockroachBinary,
"init",
secureOpt,
"--host=localhost:26259",
fmt.Sprintf("--host=localhost:%d", hostPort),
}

states := make([]int, serverArgs.numNodes)
Expand Down
42 changes: 38 additions & 4 deletions testserver/testserver_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,13 +186,38 @@ func TestRunServer(t *testing.T) {
{
name: "Insecure 3 Node",
instantiation: func(t *testing.T) (*sql.DB, func()) {
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt())
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt(),
testserver.AddListenAddrPortOpt(26257),
testserver.AddListenAddrPortOpt(26258),
testserver.AddListenAddrPortOpt(26259))
},
},
{
name: "Insecure 3 Node On Disk",
instantiation: func(t *testing.T) (*sql.DB, func()) {
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt(), testserver.StoreOnDiskOpt())
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt(),
testserver.StoreOnDiskOpt(),
testserver.AddListenAddrPortOpt(26257),
testserver.AddListenAddrPortOpt(26258),
testserver.AddListenAddrPortOpt(26259))
},
},
{
name: "Insecure 3 Node On Disk No Ports Specified",
instantiation: func(t *testing.T) (*sql.DB, func()) {
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt(),
testserver.StoreOnDiskOpt())
},
},
{
name: "Insecure 3 Node With Http Ports",
instantiation: func(t *testing.T) (*sql.DB, func()) {
return testserver.NewDBForTest(t, testserver.ThreeNodeOpt(),
testserver.StoreOnDiskOpt(),
testserver.AddHttpPortOpt(8080),
testserver.AddHttpPortOpt(8081),
testserver.AddHttpPortOpt(8082),
)
},
},
} {
Expand Down Expand Up @@ -348,7 +373,12 @@ func TestFlockOnDownloadedCRDB(t *testing.T) {
}

func TestRestartNode(t *testing.T) {
ts, err := testserver.NewTestServer(testserver.ThreeNodeOpt(), testserver.StoreOnDiskOpt())
ts, err := testserver.NewTestServer(
testserver.ThreeNodeOpt(),
testserver.StoreOnDiskOpt(),
testserver.AddListenAddrPortOpt(26257),
testserver.AddListenAddrPortOpt(26258),
testserver.AddListenAddrPortOpt(26259))
require.NoError(t, err)
defer ts.Stop()
for i := 0; i < 3; i++ {
Expand Down Expand Up @@ -440,14 +470,15 @@ func TestUpgradeNode(t *testing.T) {
}

defer func() {
require.NoError(t, exec.Command("rm", "-rf", "./temp_binaries").Start())
require.NoError(t, exec.Command("rm", "-rf", "./temp_binaries").Run())
}()

getBinary(oldVersionBinary)
getBinary(newVersionBinary)

absPathOldBinary, err := filepath.Abs(fmt.Sprintf("./temp_binaries/%s/cockroach", oldVersionBinary))
require.NoError(t, err)

absPathNewBinary, err := filepath.Abs(fmt.Sprintf("./temp_binaries/%s/cockroach", newVersionBinary))
require.NoError(t, err)

Expand All @@ -456,6 +487,9 @@ func TestUpgradeNode(t *testing.T) {
testserver.CockroachBinaryPathOpt(absPathOldBinary),
testserver.UpgradeCockroachBinaryPathOpt(absPathNewBinary),
testserver.StoreOnDiskOpt(),
testserver.AddListenAddrPortOpt(26257),
testserver.AddListenAddrPortOpt(26258),
testserver.AddListenAddrPortOpt(26259),
)
require.NoError(t, err)
defer ts.Stop()
Expand Down