Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks #15731

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks #15731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/common/tests_dmg_helpers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* (C) Copyright 2020-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -1063,7 +1064,9 @@ dmg_pool_target(const char *cmd, const char *dmg_config_file, const uuid_t uuid,
D_GOTO(out, rc = -DER_NOMEM);
}

args = cmd_push_arg(args, &argcount, "--rank=%d ", rank);
// Exclude, drain and reintegrate take ranks option which can be either a rank-list range or
// a single rank identifier.
args = cmd_push_arg(args, &argcount, "--ranks=%d ", rank);
if (args == NULL)
D_GOTO(out, rc = -DER_NOMEM);

Expand Down
4 changes: 1 addition & 3 deletions src/control/cmd/dmg/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,8 @@ func TestDmg_JsonOutput(t *testing.T) {
testArgs = append(testArgs, test.MockUUID(), "label:foo")
case "pool get-prop":
testArgs = append(testArgs, test.MockUUID(), "label")
case "pool extend":
case "pool extend", "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--ranks", "0")
case "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0")
case "pool query-targets":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0", "--target-idx", "1,3,5,7")
case "container set-owner":
Expand Down
66 changes: 40 additions & 26 deletions src/control/cmd/dmg/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ type PoolCmd struct {
Destroy poolDestroyCmd `command:"destroy" description:"Destroy a DAOS pool"`
Evict poolEvictCmd `command:"evict" description:"Evict all pool connections to a DAOS pool"`
List poolListCmd `command:"list" alias:"ls" description:"List DAOS pools"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks."`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a rank"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a rank"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks"`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a set of ranks"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a set of ranks"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a set of rank"`
Query poolQueryCmd `command:"query" description:"Query a DAOS pool"`
QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"`
GetACL poolGetACLCmd `command:"get-acl" description:"Get a DAOS pool's Access Control List"`
Expand Down Expand Up @@ -531,11 +531,17 @@ func (cmd *poolEvictCmd) Execute(args []string) error {
return err
}

// poolRanksCmd is used as an embedded command type that enables multiple ranks on a pool to be
// processed.
type poolRanksCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of rank-range strings to operate on for a single pool"`
}

// poolExcludeCmd is the struct representing the command to exclude a DAOS target.
type poolExcludeCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from each rank"`
}

// Execute is run when PoolExcludeCmd subcommand is activated
Expand All @@ -547,7 +553,11 @@ func (cmd *poolExcludeCmd) Execute(args []string) error {
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList}
req := &control.PoolExcludeReq{
ID: cmd.PoolID().String(),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
if err != nil {
Expand All @@ -561,41 +571,46 @@ func (cmd *poolExcludeCmd) Execute(args []string) error {

// poolDrainCmd is the struct representing the command to Drain a DAOS target.
type poolDrainCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be drained"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on each rank"`
}

// Execute is run when PoolDrainCmd subcommand is activated
func (cmd *poolDrainCmd) Execute(args []string) error {
msg := "succeeded"

var idxList []uint32
if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
err = errors.WithMessage(err, "parsing target list")
return err
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolDrainReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)
resp, err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)

if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(resp, err)
}

// Retrieve PoolRanksResults so we can pretty print output.
results, err := resp.GetResults(err)
if err != nil {
msg = errors.WithMessage(err, "failed").Error()
cmd.Errorf(errors.WithMessage(err, "Pool drain failed").Error())
return err
}

cmd.Infof("Drain command %s\n", msg)
var out strings.Builder
pretty.PrintPoolRankResults(&out, "drain", results)
cmd.Info(out.String())

return err
return resp.Errors()
}

// poolExtendCmd is the struct representing the command to Extend a DAOS pool.
type poolExtendCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of ranks to add to the pool"`
poolRanksCmd
}

// Execute is run when PoolExtendCmd subcommand is activated
Expand All @@ -619,9 +634,8 @@ func (cmd *poolExtendCmd) Execute(args []string) error {

// poolReintegrateCmd is the struct representing the command to Add a DAOS target.
type poolReintegrateCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be reintegrated"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into each rank"`
}

// Execute is run when poolReintegrateCmd subcommand is activated
Expand All @@ -636,7 +650,7 @@ func (cmd *poolReintegrateCmd) Execute(args []string) error {

req := &control.PoolReintegrateReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

Expand Down
37 changes: 19 additions & 18 deletions src/control/cmd/dmg/pool_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2019-2024 Intel Corporation.
// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -615,71 +616,71 @@ func TestPoolCommands(t *testing.T) {
},
{
"Exclude a target with single target idx",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Exclude a target with multiple idx",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Exclude a target with no idx given",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
strings.Join([]string{
printRequest(t, &control.PoolExcludeReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{},
}),
}, " "),
nil,
},
{
"Drain a target with single target idx",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Drain a target with multiple idx",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Drain a target with no idx given",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
strings.Join([]string{
printRequest(t, &control.PoolDrainReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1, 2},
TargetIdx: []uint32{},
}),
}, " "),
Expand Down Expand Up @@ -716,35 +717,35 @@ func TestPoolCommands(t *testing.T) {
},
{
"Reintegrate a target with single target idx",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{1},
}),
}, " "),
nil,
},
{
"Reintegrate a target with multiple idx",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1,2,3",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{1, 2, 3},
}),
}, " "),
nil,
},
{
"Reintegrate a target with no idx given",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1",
strings.Join([]string{
printRequest(t, &control.PoolReintegrateReq{
ID: "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
Rank: 0,
Ranks: []ranklist.Rank{0, 1},
TargetIdx: []uint32{},
}),
}, " "),
Expand Down
8 changes: 4 additions & 4 deletions src/control/cmd/dmg/pretty/system.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func PrintSystemCleanupResponse(out io.Writer, resp *control.SystemCleanupResp,

// PrintPoolRankResults generates a table showing results of operations on pool ranks. Each row will
// indicate a result for a group of ranks on a pool.
func PrintPoolRankResults(out io.Writer, results []*control.PoolRankResult) {
func PrintPoolRankResults(out io.Writer, opStr string, results []*control.PoolRanksResult) {
if len(results) == 0 {
fmt.Fprintln(out, "No pool ranks processed")
return
Expand All @@ -236,14 +236,14 @@ func PrintPoolRankResults(out io.Writer, results []*control.PoolRankResult) {

var table []txtfmt.TableRow
for _, r := range results {
result := "OK"
result := fmt.Sprintf("%s OK", opStr)
reason := "-"
if r.Status != 0 {
result = "FAIL"
result = fmt.Sprintf("%s FAIL", opStr)
reason = r.Msg
}
row := txtfmt.TableRow{
"Pool": r.PoolID,
"Pool": r.ID,
"Ranks": r.Ranks,
"Result": result,
"Reason": reason,
Expand Down
40 changes: 22 additions & 18 deletions src/control/cmd/dmg/pretty/system_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,37 +615,41 @@ Unknown 3 hosts: foo[7-9]

func TestPretty_PrintPoolRankResults(t *testing.T) {
for name, tc := range map[string]struct {
results []*control.PoolRankResult
op string
results []*control.PoolRanksResult
expOut string
}{
"normal response": {
results: []*control.PoolRankResult{
op: "drain",
results: []*control.PoolRanksResult{
{PoolID: test.MockUUID(1), Ranks: "0-3"},
{PoolID: test.MockUUID(2), Ranks: "1-4"},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 0-3 OK -
00000002-0002-0002-0002-000000000002 1-4 OK -
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 0-3 drain OK -
00000002-0002-0002-0002-000000000002 1-4 drain OK -

`,
},
"normal response; use labels": {
results: []*control.PoolRankResult{
op: "drain",
results: []*control.PoolRanksResult{
{PoolID: "label1", Ranks: "0-3"},
{PoolID: "label2", Ranks: "1-4"},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
label1 0-3 OK -
label2 1-4 OK -
Pool Ranks Result Reason
---- ----- ------ ------
label1 0-3 drain OK -
label2 1-4 drain OK -

`,
},
"response with failures": {
results: []*control.PoolRankResult{
op: "reintegrate",
results: []*control.PoolRanksResult{
{PoolID: test.MockUUID(1), Ranks: "1-2"},
{PoolID: test.MockUUID(2), Ranks: "0"},
{
Expand All @@ -654,18 +658,18 @@ label2 1-4 OK -
},
},
expOut: `
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 1-2 OK -
00000002-0002-0002-0002-000000000002 0 OK -
00000002-0002-0002-0002-000000000002 1-2 FAIL fail1
Pool Ranks Result Reason
---- ----- ------ ------
00000001-0001-0001-0001-000000000001 1-2 reintegrate OK -
00000002-0002-0002-0002-000000000002 0 reintegrate OK -
00000002-0002-0002-0002-000000000002 1-2 reintegrate FAIL fail1

`,
},
} {
t.Run(name, func(t *testing.T) {
var out strings.Builder
PrintPoolRankResults(&out, tc.results)
PrintPoolRankResults(&out, tc.op, tc.results)

if diff := cmp.Diff(strings.TrimLeft(tc.expOut, "\n"), out.String()); diff != "" {
t.Fatalf("unexpected stdout (-want, +got):\n%s\n", diff)
Expand Down
Loading