Skip to content

Commit

Permalink
DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks
Browse files Browse the repository at this point in the history 
Signed-off-by: Tom Nabarro <thomas.nabarro@hpe.com>
  • Loading branch information
@tanabarr
tanabarr committed Jan 22, 2025
1 parent 0be2d10 commit 9352123
Show file tree
Hide file tree
Showing 32 changed files with 1,738 additions and 1,744 deletions.
5 changes: 4 additions & 1 deletion src/common/tests_dmg_helpers.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* (C) Copyright 2020-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -1063,7 +1064,9 @@ dmg_pool_target(const char *cmd, const char *dmg_config_file, const uuid_t uuid,
D_GOTO(out, rc = -DER_NOMEM);
}

args = cmd_push_arg(args, &argcount, "--rank=%d ", rank);
// Exclude, drain and reintegrate take ranks option which can be either a rank-list range or
// a single rank identifier.
args = cmd_push_arg(args, &argcount, "--ranks=%d ", rank);
if (args == NULL)
D_GOTO(out, rc = -DER_NOMEM);

Expand Down
4 changes: 1 addition & 3 deletions src/control/cmd/dmg/json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,8 @@ func TestDmg_JsonOutput(t *testing.T) {
testArgs = append(testArgs, test.MockUUID(), "label:foo")
case "pool get-prop":
testArgs = append(testArgs, test.MockUUID(), "label")
case "pool extend":
case "pool extend", "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--ranks", "0")
case "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0")
case "pool query-targets":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0", "--target-idx", "1,3,5,7")
case "container set-owner":
Expand Down
66 changes: 40 additions & 26 deletions src/control/cmd/dmg/pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ type PoolCmd struct {
Destroy poolDestroyCmd `command:"destroy" description:"Destroy a DAOS pool"`
Evict poolEvictCmd `command:"evict" description:"Evict all pool connections to a DAOS pool"`
List poolListCmd `command:"list" alias:"ls" description:"List DAOS pools"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks."`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a rank"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a rank"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks"`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a set of ranks"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a set of ranks"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a set of rank"`
Query poolQueryCmd `command:"query" description:"Query a DAOS pool"`
QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"`
GetACL poolGetACLCmd `command:"get-acl" description:"Get a DAOS pool's Access Control List"`
Expand Down Expand Up @@ -531,11 +531,17 @@ func (cmd *poolEvictCmd) Execute(args []string) error {
return err
}

// poolRanksCmd is used as an embedded command type that enables multiple ranks on a pool to be
// processed.
type poolRanksCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of rank-range strings to operate on for a single pool"`
}

// poolExcludeCmd is the struct representing the command to exclude a DAOS target.
type poolExcludeCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from each rank"`
}

// Execute is run when PoolExcludeCmd subcommand is activated
Expand All @@ -547,7 +553,11 @@ func (cmd *poolExcludeCmd) Execute(args []string) error {
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList}
req := &control.PoolExcludeReq{
ID: cmd.PoolID().String(),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
if err != nil {
Expand All @@ -561,41 +571,46 @@ func (cmd *poolExcludeCmd) Execute(args []string) error {

// poolDrainCmd is the struct representing the command to Drain a DAOS target.
type poolDrainCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be drained"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on each rank"`
}

// Execute is run when PoolDrainCmd subcommand is activated
func (cmd *poolDrainCmd) Execute(args []string) error {
msg := "succeeded"

var idxList []uint32
if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
err = errors.WithMessage(err, "parsing target list")
return err
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolDrainReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)
resp, err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)

if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(resp, err)
}

// Retrieve PoolRanksResults so we can pretty print output.
results, err := resp.GetResults(err)
if err != nil {
msg = errors.WithMessage(err, "failed").Error()
cmd.Errorf(errors.WithMessage(err, "Pool drain failed").Error())
return err
}

cmd.Infof("Drain command %s\n", msg)
var out strings.Builder
pretty.PrintPoolRankResults(&out, "drain", results)
cmd.Info(out.String())

return err
return resp.Errors()
}

// poolExtendCmd is the struct representing the command to Extend a DAOS pool.
type poolExtendCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of ranks to add to the pool"`
poolRanksCmd
}

// Execute is run when PoolExtendCmd subcommand is activated
Expand All @@ -619,9 +634,8 @@ func (cmd *poolExtendCmd) Execute(args []string) error {

// poolReintegrateCmd is the struct representing the command to Add a DAOS target.
type poolReintegrateCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be reintegrated"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into each rank"`
}

// Execute is run when poolReintegrateCmd subcommand is activated
Expand All @@ -636,7 +650,7 @@ func (cmd *poolReintegrateCmd) Execute(args []string) error {

req := &control.PoolReintegrateReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

Expand Down
37 changes: 19 additions & 18 deletions src/control/cmd/dmg/pool_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2019-2024 Intel Corporation.
// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -615,71 +616,71 @@ func TestPoolCommands(t *testing.T) {
},
{
"Exclude a target with single target idx",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Exclude a target with multiple idx",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Exclude a target with no idx given",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{},
}),
}, " "),
nil,
},
{
"Drain a target with single target idx",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Drain a target with multiple idx",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Drain a target with no idx given",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{},
}),
}, " "),
Expand Down Expand Up @@ -716,35 +717,35 @@ func TestPoolCommands(t *testing.T) {
},
{
"Reintegrate a target with single target idx",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Reintegrate a target with multiple idx",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Reintegrate a target with no idx given",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{},
}),
}, " "),
Expand Down
8 changes: 4 additions & 4 deletions src/control/cmd/dmg/pretty/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func PrintSystemCleanupResponse(out io.Writer, resp *control.SystemCleanupResp,

// PrintPoolRankResults generates a table showing results of operations on pool ranks. Each row will
// indicate a result for a group of ranks on a pool.
func PrintPoolRankResults(out io.Writer, results []*control.PoolRankResult) {
func PrintPoolRankResults(out io.Writer, opStr string, results []*control.PoolRanksResult) {
if len(results) == 0 {
fmt.Fprintln(out, "No pool ranks processed")
return
Expand All @@ -236,14 +236,14 @@ func PrintPoolRankResults(out io.Writer, results []*control.PoolRankResult) {

var table []txtfmt.TableRow
for _, r := range results {
result := "OK"
result := fmt.Sprintf("%s OK", opStr)
reason := "-"
if r.Status != 0 {
result = "FAIL"
result = fmt.Sprintf("%s FAIL", opStr)
reason = r.Msg
}
row := txtfmt.TableRow{
"Pool": r.PoolID,
"Pool": r.ID,
"Ranks": r.Ranks,
"Result": result,
"Reason": reason,
Expand Down
40 changes: 22 additions & 18 deletions src/control/cmd/dmg/pretty/system_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,37 +615,41 @@ Unknown 3 hosts: foo[7-9]

func TestPretty_PrintPoolRankResults(t *testing.T) {
for name, tc := range map[string]struct {
results []*control.PoolRankResult
op string
results []*control.PoolRanksResult
expOut string
}{
"normal response": {
results: []*control.PoolRankResult{
op: "drain",
results: []*control.PoolRanksResult{
{PoolID: test.MockUUID(1), Ranks: "0-3"},
{PoolID: test.MockUUID(2), Ranks: "1-4"},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 0-3 OK -
00000002-0002-0002-0002-000000000002 1-4 OK -
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 0-3 drain OK -
00000002-0002-0002-0002-000000000002 1-4 drain OK -
`,
},
"normal response; use labels": {
results: []*control.PoolRankResult{
op: "drain",
results: []*control.PoolRanksResult{
{PoolID: "label1", Ranks: "0-3"},
{PoolID: "label2", Ranks: "1-4"},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
label1 0-3 OK -
label2 1-4 OK -
Pool Ranks Result Reason
---- ----- ------ ------
label1 0-3 drain OK -
label2 1-4 drain OK -
`,
},
"response with failures": {
results: []*control.PoolRankResult{
op: "reintegrate",
results: []*control.PoolRanksResult{
{PoolID: test.MockUUID(1), Ranks: "1-2"},
{PoolID: test.MockUUID(2), Ranks: "0"},
{
Expand All @@ -654,18 +658,18 @@ label2 1-4 OK -
},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 1-2 OK -
00000002-0002-0002-0002-000000000002 0 OK -
00000002-0002-0002-0002-000000000002 1-2 FAIL fail1
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 1-2 reintegrate OK -
00000002-0002-0002-0002-000000000002 0 reintegrate OK -
00000002-0002-0002-0002-000000000002 1-2 reintegrate FAIL fail1
`,
},
} {
t.Run(name, func(t *testing.T) {
var out strings.Builder
PrintPoolRankResults(&out, tc.results)
PrintPoolRankResults(&out, tc.op, tc.results)

if diff := cmp.Diff(strings.TrimLeft(tc.expOut, "\n"), out.String()); diff != "" {
t.Fatalf("unexpected stdout (-want, +got):\n%s\n", diff)
Expand Down
Loading

0 comments on commit 9352123

Please sign in to comment.