From 488acf75cc3ba1bc276e499b86580ab352441d7c Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Sat, 16 Nov 2024 08:09:37 +0000 Subject: [PATCH 1/6] DAOS-7485 control: Implement dmg system drain to act on all hosts Required-githooks: true Signed-off-by: Tom Nabarro --- docs/admin/pool_operations.md | 4 +- src/control/cmd/dmg/command_test.go | 2 + src/control/cmd/dmg/json_test.go | 4 +- src/control/cmd/dmg/pool.go | 6 +- src/control/cmd/dmg/pretty/system.go | 55 +- src/control/cmd/dmg/system.go | 135 ++-- src/control/common/proto/mgmt/mgmt.pb.go | 390 ++++++------ src/control/common/proto/mgmt/mgmt_grpc.pb.go | 41 +- src/control/common/proto/mgmt/system.pb.go | 595 ++++++++++++------ src/control/lib/control/pool.go | 7 +- src/control/lib/control/response_test.go | 6 +- src/control/lib/control/system.go | 174 +++-- src/control/security/grpc_authorization.go | 3 +- .../security/grpc_authorization_test.go | 3 +- src/control/server/mgmt_system.go | 107 +++- src/mgmt/srv_drpc.c | 1 + src/proto/mgmt/mgmt.proto | 2 + src/proto/mgmt/system.proto | 26 +- 18 files changed, 1063 insertions(+), 498 deletions(-) diff --git a/docs/admin/pool_operations.md b/docs/admin/pool_operations.md index 169d89372c9..d423f6a9b65 100644 --- a/docs/admin/pool_operations.md +++ b/docs/admin/pool_operations.md @@ -591,8 +591,8 @@ with the following information for each pool: - The imbalance percentage indicating whether data distribution across the difference storage targets is well balanced. 0% means that there is no imbalance and 100% means that out-of-space errors might be returned - by some storage targets while space is still available on others. Again - for the NVMe or DATA tier. + by some storage targets while space is still available on others. Applies + only for the NVMe or DATA tier. - The number of disabled targets (0 here) and the number of targets that the pool was originally configured with (total). diff --git a/src/control/cmd/dmg/command_test.go b/src/control/cmd/dmg/command_test.go index f56714dfaee..8f70322f328 100644 --- a/src/control/cmd/dmg/command_test.go +++ b/src/control/cmd/dmg/command_test.go @@ -134,6 +134,8 @@ func (bci *bridgeConnInvoker) InvokeUnaryRPC(ctx context.Context, uReq control.U resp = control.MockMSResponse("", nil, &mgmtpb.SystemStartResp{}) case *control.SystemExcludeReq: resp = control.MockMSResponse("", nil, &mgmtpb.SystemExcludeResp{}) + case *control.SystemDrainReq: + resp = control.MockMSResponse("", nil, &mgmtpb.SystemDrainResp{}) case *control.SystemQueryReq: if req.FailOnUnavailable { resp = control.MockMSResponse("", system.ErrRaftUnavail, nil) diff --git a/src/control/cmd/dmg/json_test.go b/src/control/cmd/dmg/json_test.go index 9021f1c8d5d..84fc87a7293 100644 --- a/src/control/cmd/dmg/json_test.go +++ b/src/control/cmd/dmg/json_test.go @@ -113,9 +113,7 @@ func TestDmg_JsonOutput(t *testing.T) { testArgs = append(testArgs, "foo:bar") case "system del-attr": testArgs = append(testArgs, "foo") - case "system exclude": - testArgs = append(testArgs, "--ranks", "0") - case "system clear-exclude": + case "system exclude", "system clear-exclude", "system drain": testArgs = append(testArgs, "--ranks", "0") } diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index dbb9f50e495..5ca56a18244 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -575,7 +575,11 @@ func (cmd *PoolDrainCmd) Execute(args []string) error { return err } - req := &control.PoolDrainReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList} + req := &control.PoolDrainReq{ + ID: cmd.PoolID().String(), + Rank: ranklist.Rank(cmd.Rank), + TargetIdx: idxList, + } err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req) if err != nil { diff --git a/src/control/cmd/dmg/pretty/system.go b/src/control/cmd/dmg/pretty/system.go index 29cd0d0956e..8f7c7296cc2 100644 --- a/src/control/cmd/dmg/pretty/system.go +++ b/src/control/cmd/dmg/pretty/system.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -229,3 +229,56 @@ func PrintSystemCleanupResponse(out, outErr io.Writer, resp *control.SystemClean fmt.Fprintln(out, "System Cleanup Success") return nil } + +func printSystemDrainRespVerbose(out io.Writer, resp *control.SystemDrainResp) error { + if len(resp.Results) == 0 { + fmt.Fprintln(out, "no pool ranks drained") + return nil + } + + titles := []string{"Pool", "Ranks Drained"} + formatter := txtfmt.NewTableFormatter(titles...) + + var table []txtfmt.TableRow + for _, r := range resp.Results { + row := txtfmt.TableRow{ + "Pool": r.PoolID, + "Ranks Drained": r.Ranks, + } + table = append(table, row) + } + + fmt.Fprintln(out, formatter.Format(table)) + + return nil +} + +// PrintSystemDrainResponse generates a human-readable representation of the +// supplied SystemDrainResp struct and writes it to the supplied io.Writer. +func PrintSystemDrainResponse(out, outErr io.Writer, resp *control.SystemDrainResp, verbose bool) error { + err := resp.Errors() + + if err != nil { + fmt.Fprintln(outErr, err.Error()) + } + + if len(resp.Results) == 0 { + fmt.Fprintln(out, "No pool ranks drained") + return nil + } + + if verbose { + return printSystemDrainRespVerbose(out, resp) + } + + var msg string + for i, result := range resp.Results { + if i != 0 { + fmt.Sprintf("%s, ", msg) + } + fmt.Sprintf("%s%s: %s", msg, result.PoolID, result.Ranks) + } + + fmt.Fprintf(out, "System Drain Success. Drained pool ranks: %s\n", msg) + return nil +} diff --git a/src/control/cmd/dmg/system.go b/src/control/cmd/dmg/system.go index 376b1c4e5f8..8dafb947f58 100644 --- a/src/control/cmd/dmg/system.go +++ b/src/control/cmd/dmg/system.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -31,9 +31,10 @@ type SystemCmd struct { Start systemStartCmd `command:"start" description:"Perform start of stopped DAOS system"` Exclude systemExcludeCmd `command:"exclude" description:"Exclude ranks from DAOS system"` ClearExclude systemClearExcludeCmd `command:"clear-exclude" description:"Clear excluded state for ranks"` + Drain systemDrainCmd `command:"drain" description:"Drain ranks or hosts from all relevant pools in DAOS system"` Erase systemEraseCmd `command:"erase" description:"Erase system metadata prior to reformat"` ListPools PoolListCmd `command:"list-pools" description:"List all pools in the DAOS system"` - Cleanup systemCleanupCmd `command:"cleanup" description:"Clean up all resources associated with the specified machine"` + Cleanu systemCleanupCmd `command:"cleanup" description:"Clean up all resources associated with the specified machine"` SetAttr systemSetAttrCmd `command:"set-attr" description:"Set system attributes"` GetAttr systemGetAttrCmd `command:"get-attr" description:"Get system attributes"` DelAttr systemDelAttrCmd `command:"del-attr" description:"Delete system attributes"` @@ -41,11 +42,15 @@ type SystemCmd struct { GetProp systemGetPropCmd `command:"get-prop" description:"Get system properties"` } -type leaderQueryCmd struct { +type baseCtlCmd struct { baseCmd cfgCmd ctlInvokerCmd cmdutil.JSONOutputCmd +} + +type leaderQueryCmd struct { + baseCtlCmd DownReplicas bool `short:"N" long:"down-replicas" description:"Show Down Replicas only"` } @@ -98,13 +103,14 @@ func (cmd *rankListCmd) validateHostsRanks() error { return nil } +type baseRankListCmd struct { + baseCtlCmd + rankListCmd +} + // systemQueryCmd is the struct representing the command to query system status. type systemQueryCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - rankListCmd + baseRankListCmd Verbose bool `long:"verbose" short:"v" description:"Display more member details"` NotOK bool `long:"not-ok" description:"Display components in need of administrative investigation"` WantedStates ui.MemberStateSetFlag `long:"with-states" description:"Only show engines in one of a set of comma-separated states"` @@ -166,11 +172,7 @@ func (cmd *systemEraseCmd) Execute(_ []string) error { // systemStopCmd is the struct representing the command to shutdown DAOS system. type systemStopCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - rankListCmd + baseRankListCmd Force bool `long:"force" description:"Force stop DAOS system members"` } @@ -210,15 +212,11 @@ func (cmd *systemStopCmd) Execute(_ []string) (errOut error) { return resp.Errors() } -type baseExcludeCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - rankListCmd +type systemExcludeCmd struct { + baseRankListCmd } -func (cmd *baseExcludeCmd) execute(clear bool) error { +func (cmd *systemExcludeCmd) execute(clear bool) error { if err := cmd.validateHostsRanks(); err != nil { return err } @@ -252,29 +250,67 @@ func (cmd *baseExcludeCmd) execute(clear bool) error { return nil } -type systemExcludeCmd struct { - baseExcludeCmd -} - func (cmd *systemExcludeCmd) Execute(_ []string) error { return cmd.execute(false) } type systemClearExcludeCmd struct { - baseExcludeCmd + systemExcludeCmd } func (cmd *systemClearExcludeCmd) Execute(_ []string) error { return cmd.execute(true) } +type systemDrainCmd struct { + baseRankListCmd + Verbose bool `long:"verbose" short:"v" description:"Output additional system drain information"` +} + +func (cmd *systemDrainCmd) Execute(_ []string) (errOut error) { + defer func() { + errOut = errors.Wrap(errOut, "system drain failed") + }() + + if err := cmd.validateHostsRanks(); err != nil { + return err + } + if cmd.Ranks.Count() == 0 && cmd.Hosts.Count() == 0 { + return errors.New("no ranks or hosts specified") + } + + req := new(control.SystemDrainReq) + req.SetSystem(cmd.config.SystemName) + req.Hosts.Replace(&cmd.Hosts.HostSet) + req.Ranks.Replace(&cmd.Ranks.RankSet) + + resp, err := control.SystemDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req) + if err != nil { + return err // control api returned an error, disregard response + } + + if cmd.JSONOutputEnabled() { + return cmd.OutputJSON(resp, resp.Errors()) + } + + var out, outErr strings.Builder + if err := pretty.PrintSystemDrainResponse(&out, &outErr, resp, cmd.Verbose); err != nil { + return err + } + if outErr.String() != "" { + cmd.Error(outErr.String()) + } + + // Infof prints raw string and doesn't try to expand "%" + // preserving column formatting in txtfmt table + cmd.Infof("%s", out.String()) + + return nil +} + // systemStartCmd is the struct representing the command to start system. type systemStartCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - rankListCmd + baseRankListCmd } // Execute is run when systemStartCmd activates. @@ -312,15 +348,10 @@ func (cmd *systemStartCmd) Execute(_ []string) (errOut error) { } type systemCleanupCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Machine string `positional-arg-name:""` } `positional-args:"yes"` - Verbose bool `long:"verbose" short:"v" description:"Output additional cleanup information"` } @@ -340,7 +371,7 @@ func (cmd *systemCleanupCmd) Execute(_ []string) (errOut error) { } if cmd.JSONOutputEnabled() { - return cmd.OutputJSON(resp, err) + return cmd.OutputJSON(resp, resp.Errors()) } var out, outErr strings.Builder @@ -360,11 +391,7 @@ func (cmd *systemCleanupCmd) Execute(_ []string) (errOut error) { // systemSetAttrCmd represents the command to set system attributes. type systemSetAttrCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Attrs ui.SetPropertiesFlag `positional-arg-name:"system attributes to set (key:val[,key:val...])" required:"1"` } `positional-args:"yes"` @@ -391,11 +418,7 @@ func (cmd *systemSetAttrCmd) Execute(_ []string) error { // systemGetAttrCmd represents the command to get system attributes. type systemGetAttrCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Attrs ui.GetPropertiesFlag `positional-arg-name:"system attributes to get (key[,key...])"` } `positional-args:"yes"` @@ -446,11 +469,7 @@ func (cmd *systemGetAttrCmd) Execute(_ []string) error { // systemDelAttrCmd represents the command to delete system attributes. type systemDelAttrCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Attrs ui.GetPropertiesFlag `positional-arg-name:"system attributes to delete (key[,key...])" required:"1"` } `positional-args:"yes"` @@ -525,11 +544,7 @@ func (f *systemSetPropsFlag) Complete(match string) []flags.Completion { // systemSetPropCmd represents the command to set system properties. type systemSetPropCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Props systemSetPropsFlag `positional-arg-name:"system properties to set (key:val[,key:val...])" required:"1"` } `positional-args:"yes"` @@ -597,11 +612,7 @@ func (f *systemGetPropsFlag) Complete(match string) []flags.Completion { // systemGetPropCmd represents the command to get system properties. type systemGetPropCmd struct { - baseCmd - cfgCmd - ctlInvokerCmd - cmdutil.JSONOutputCmd - + baseCtlCmd Args struct { Props systemGetPropsFlag `positional-arg-name:"system properties to get (key[,key...])"` } `positional-args:"yes"` diff --git a/src/control/common/proto/mgmt/mgmt.pb.go b/src/control/common/proto/mgmt/mgmt.pb.go index 0b45d1cae25..24da4c98de9 100644 --- a/src/control/common/proto/mgmt/mgmt.pb.go +++ b/src/control/common/proto/mgmt/mgmt.pb.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -41,7 +41,7 @@ var file_mgmt_mgmt_proto_rawDesc = []byte{ 0x11, 0x6d, 0x67, 0x6d, 0x74, 0x2f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0d, 0x63, 0x68, 0x6b, 0x2f, 0x63, 0x68, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x10, 0x63, 0x68, 0x6b, 0x2f, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x32, 0x90, 0x15, 0x0a, 0x07, 0x4d, 0x67, 0x6d, 0x74, 0x53, 0x76, 0x63, 0x12, + 0x6f, 0x74, 0x6f, 0x32, 0xce, 0x15, 0x0a, 0x07, 0x4d, 0x67, 0x6d, 0x74, 0x53, 0x76, 0x63, 0x12, 0x27, 0x0a, 0x04, 0x4a, 0x6f, 0x69, 0x6e, 0x12, 0x0d, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x4a, 0x6f, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0c, 0x43, 0x6c, 0x75, 0x73, @@ -139,82 +139,86 @@ var file_mgmt_mgmt_proto_rawDesc = []byte{ 0x64, 0x65, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, - 0x72, 0x61, 0x73, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, - 0x61, 0x6e, 0x75, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, - 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, + 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, + 0x73, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, + 0x75, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x12, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x1a, + 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, 0x6d, + 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0f, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x41, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x12, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x15, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, - 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0f, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, - 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x13, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, - 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x70, 0x61, 0x69, 0x72, 0x12, 0x11, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, - 0x00, 0x12, 0x3c, 0x0a, 0x0b, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x12, 0x14, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, - 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, - 0x39, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, - 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, - 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x12, 0x16, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, - 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, - 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, - 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, - 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, - 0x11, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x12, 0x10, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x14, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, - 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, - 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, - 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x18, - 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x67, 0x6d, 0x74, 0x50, - 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, - 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, - 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, - 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x70, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x14, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x17, 0x2e, 0x6d, + 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, + 0x63, 0x79, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x47, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x73, 0x70, 0x22, + 0x00, 0x12, 0x3c, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x70, 0x61, 0x69, 0x72, 0x12, 0x11, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, + 0x3c, 0x0a, 0x0b, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, 0x0a, + 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x12, 0x16, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, + 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, + 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, + 0x71, 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, + 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, 0x16, 0x2e, + 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, + 0x6f, 0x70, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0d, 0x53, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x12, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, + 0x1a, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, + 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x11, 0x46, + 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x12, 0x10, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x14, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, + 0x65, 0x63, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, 0x2e, 0x63, + 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x18, 0x46, 0x61, + 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x67, 0x6d, 0x74, 0x50, 0x6f, 0x6f, + 0x6c, 0x46, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x0a, 0x2e, 0x63, 0x68, 0x6b, 0x2e, 0x46, 0x61, 0x75, + 0x6c, 0x74, 0x1a, 0x0e, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x44, 0x61, 0x6f, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x00, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, + 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, + 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var file_mgmt_mgmt_proto_goTypes = []interface{}{ @@ -243,56 +247,58 @@ var file_mgmt_mgmt_proto_goTypes = []interface{}{ (*SystemStopReq)(nil), // 22: mgmt.SystemStopReq (*SystemStartReq)(nil), // 23: mgmt.SystemStartReq (*SystemExcludeReq)(nil), // 24: mgmt.SystemExcludeReq - (*SystemEraseReq)(nil), // 25: mgmt.SystemEraseReq - (*SystemCleanupReq)(nil), // 26: mgmt.SystemCleanupReq - (*CheckEnableReq)(nil), // 27: mgmt.CheckEnableReq - (*CheckDisableReq)(nil), // 28: mgmt.CheckDisableReq - (*CheckStartReq)(nil), // 29: mgmt.CheckStartReq - (*CheckStopReq)(nil), // 30: mgmt.CheckStopReq - (*CheckQueryReq)(nil), // 31: mgmt.CheckQueryReq - (*CheckSetPolicyReq)(nil), // 32: mgmt.CheckSetPolicyReq - (*CheckGetPolicyReq)(nil), // 33: mgmt.CheckGetPolicyReq - (*CheckActReq)(nil), // 34: mgmt.CheckActReq - (*PoolUpgradeReq)(nil), // 35: mgmt.PoolUpgradeReq - (*SystemSetAttrReq)(nil), // 36: mgmt.SystemSetAttrReq - (*SystemGetAttrReq)(nil), // 37: mgmt.SystemGetAttrReq - (*SystemSetPropReq)(nil), // 38: mgmt.SystemSetPropReq - (*SystemGetPropReq)(nil), // 39: mgmt.SystemGetPropReq - (*chk.CheckReport)(nil), // 40: chk.CheckReport - (*chk.Fault)(nil), // 41: chk.Fault - (*JoinResp)(nil), // 42: mgmt.JoinResp - (*shared.ClusterEventResp)(nil), // 43: shared.ClusterEventResp - (*LeaderQueryResp)(nil), // 44: mgmt.LeaderQueryResp - (*PoolCreateResp)(nil), // 45: mgmt.PoolCreateResp - (*PoolDestroyResp)(nil), // 46: mgmt.PoolDestroyResp - (*PoolEvictResp)(nil), // 47: mgmt.PoolEvictResp - (*PoolExcludeResp)(nil), // 48: mgmt.PoolExcludeResp - (*PoolDrainResp)(nil), // 49: mgmt.PoolDrainResp - (*PoolExtendResp)(nil), // 50: mgmt.PoolExtendResp - (*PoolReintegrateResp)(nil), // 51: mgmt.PoolReintegrateResp - (*PoolQueryResp)(nil), // 52: mgmt.PoolQueryResp - (*PoolQueryTargetResp)(nil), // 53: mgmt.PoolQueryTargetResp - (*PoolSetPropResp)(nil), // 54: mgmt.PoolSetPropResp - (*PoolGetPropResp)(nil), // 55: mgmt.PoolGetPropResp - (*ACLResp)(nil), // 56: mgmt.ACLResp - (*GetAttachInfoResp)(nil), // 57: mgmt.GetAttachInfoResp - (*ListPoolsResp)(nil), // 58: mgmt.ListPoolsResp - (*ListContResp)(nil), // 59: mgmt.ListContResp - (*DaosResp)(nil), // 60: mgmt.DaosResp - (*SystemQueryResp)(nil), // 61: mgmt.SystemQueryResp - (*SystemStopResp)(nil), // 62: mgmt.SystemStopResp - (*SystemStartResp)(nil), // 63: mgmt.SystemStartResp - (*SystemExcludeResp)(nil), // 64: mgmt.SystemExcludeResp - (*SystemEraseResp)(nil), // 65: mgmt.SystemEraseResp - (*SystemCleanupResp)(nil), // 66: mgmt.SystemCleanupResp - (*CheckStartResp)(nil), // 67: mgmt.CheckStartResp - (*CheckStopResp)(nil), // 68: mgmt.CheckStopResp - (*CheckQueryResp)(nil), // 69: mgmt.CheckQueryResp - (*CheckGetPolicyResp)(nil), // 70: mgmt.CheckGetPolicyResp - (*CheckActResp)(nil), // 71: mgmt.CheckActResp - (*PoolUpgradeResp)(nil), // 72: mgmt.PoolUpgradeResp - (*SystemGetAttrResp)(nil), // 73: mgmt.SystemGetAttrResp - (*SystemGetPropResp)(nil), // 74: mgmt.SystemGetPropResp + (*SystemDrainReq)(nil), // 25: mgmt.SystemDrainReq + (*SystemEraseReq)(nil), // 26: mgmt.SystemEraseReq + (*SystemCleanupReq)(nil), // 27: mgmt.SystemCleanupReq + (*CheckEnableReq)(nil), // 28: mgmt.CheckEnableReq + (*CheckDisableReq)(nil), // 29: mgmt.CheckDisableReq + (*CheckStartReq)(nil), // 30: mgmt.CheckStartReq + (*CheckStopReq)(nil), // 31: mgmt.CheckStopReq + (*CheckQueryReq)(nil), // 32: mgmt.CheckQueryReq + (*CheckSetPolicyReq)(nil), // 33: mgmt.CheckSetPolicyReq + (*CheckGetPolicyReq)(nil), // 34: mgmt.CheckGetPolicyReq + (*CheckActReq)(nil), // 35: mgmt.CheckActReq + (*PoolUpgradeReq)(nil), // 36: mgmt.PoolUpgradeReq + (*SystemSetAttrReq)(nil), // 37: mgmt.SystemSetAttrReq + (*SystemGetAttrReq)(nil), // 38: mgmt.SystemGetAttrReq + (*SystemSetPropReq)(nil), // 39: mgmt.SystemSetPropReq + (*SystemGetPropReq)(nil), // 40: mgmt.SystemGetPropReq + (*chk.CheckReport)(nil), // 41: chk.CheckReport + (*chk.Fault)(nil), // 42: chk.Fault + (*JoinResp)(nil), // 43: mgmt.JoinResp + (*shared.ClusterEventResp)(nil), // 44: shared.ClusterEventResp + (*LeaderQueryResp)(nil), // 45: mgmt.LeaderQueryResp + (*PoolCreateResp)(nil), // 46: mgmt.PoolCreateResp + (*PoolDestroyResp)(nil), // 47: mgmt.PoolDestroyResp + (*PoolEvictResp)(nil), // 48: mgmt.PoolEvictResp + (*PoolExcludeResp)(nil), // 49: mgmt.PoolExcludeResp + (*PoolDrainResp)(nil), // 50: mgmt.PoolDrainResp + (*PoolExtendResp)(nil), // 51: mgmt.PoolExtendResp + (*PoolReintegrateResp)(nil), // 52: mgmt.PoolReintegrateResp + (*PoolQueryResp)(nil), // 53: mgmt.PoolQueryResp + (*PoolQueryTargetResp)(nil), // 54: mgmt.PoolQueryTargetResp + (*PoolSetPropResp)(nil), // 55: mgmt.PoolSetPropResp + (*PoolGetPropResp)(nil), // 56: mgmt.PoolGetPropResp + (*ACLResp)(nil), // 57: mgmt.ACLResp + (*GetAttachInfoResp)(nil), // 58: mgmt.GetAttachInfoResp + (*ListPoolsResp)(nil), // 59: mgmt.ListPoolsResp + (*ListContResp)(nil), // 60: mgmt.ListContResp + (*DaosResp)(nil), // 61: mgmt.DaosResp + (*SystemQueryResp)(nil), // 62: mgmt.SystemQueryResp + (*SystemStopResp)(nil), // 63: mgmt.SystemStopResp + (*SystemStartResp)(nil), // 64: mgmt.SystemStartResp + (*SystemExcludeResp)(nil), // 65: mgmt.SystemExcludeResp + (*SystemDrainResp)(nil), // 66: mgmt.SystemDrainResp + (*SystemEraseResp)(nil), // 67: mgmt.SystemEraseResp + (*SystemCleanupResp)(nil), // 68: mgmt.SystemCleanupResp + (*CheckStartResp)(nil), // 69: mgmt.CheckStartResp + (*CheckStopResp)(nil), // 70: mgmt.CheckStopResp + (*CheckQueryResp)(nil), // 71: mgmt.CheckQueryResp + (*CheckGetPolicyResp)(nil), // 72: mgmt.CheckGetPolicyResp + (*CheckActResp)(nil), // 73: mgmt.CheckActResp + (*PoolUpgradeResp)(nil), // 74: mgmt.PoolUpgradeResp + (*SystemGetAttrResp)(nil), // 75: mgmt.SystemGetAttrResp + (*SystemGetPropResp)(nil), // 76: mgmt.SystemGetPropResp } var file_mgmt_mgmt_proto_depIdxs = []int32{ 0, // 0: mgmt.MgmtSvc.Join:input_type -> mgmt.JoinReq @@ -321,70 +327,72 @@ var file_mgmt_mgmt_proto_depIdxs = []int32{ 22, // 23: mgmt.MgmtSvc.SystemStop:input_type -> mgmt.SystemStopReq 23, // 24: mgmt.MgmtSvc.SystemStart:input_type -> mgmt.SystemStartReq 24, // 25: mgmt.MgmtSvc.SystemExclude:input_type -> mgmt.SystemExcludeReq - 25, // 26: mgmt.MgmtSvc.SystemErase:input_type -> mgmt.SystemEraseReq - 26, // 27: mgmt.MgmtSvc.SystemCleanup:input_type -> mgmt.SystemCleanupReq - 27, // 28: mgmt.MgmtSvc.SystemCheckEnable:input_type -> mgmt.CheckEnableReq - 28, // 29: mgmt.MgmtSvc.SystemCheckDisable:input_type -> mgmt.CheckDisableReq - 29, // 30: mgmt.MgmtSvc.SystemCheckStart:input_type -> mgmt.CheckStartReq - 30, // 31: mgmt.MgmtSvc.SystemCheckStop:input_type -> mgmt.CheckStopReq - 31, // 32: mgmt.MgmtSvc.SystemCheckQuery:input_type -> mgmt.CheckQueryReq - 32, // 33: mgmt.MgmtSvc.SystemCheckSetPolicy:input_type -> mgmt.CheckSetPolicyReq - 33, // 34: mgmt.MgmtSvc.SystemCheckGetPolicy:input_type -> mgmt.CheckGetPolicyReq - 34, // 35: mgmt.MgmtSvc.SystemCheckRepair:input_type -> mgmt.CheckActReq - 35, // 36: mgmt.MgmtSvc.PoolUpgrade:input_type -> mgmt.PoolUpgradeReq - 36, // 37: mgmt.MgmtSvc.SystemSetAttr:input_type -> mgmt.SystemSetAttrReq - 37, // 38: mgmt.MgmtSvc.SystemGetAttr:input_type -> mgmt.SystemGetAttrReq - 38, // 39: mgmt.MgmtSvc.SystemSetProp:input_type -> mgmt.SystemSetPropReq - 39, // 40: mgmt.MgmtSvc.SystemGetProp:input_type -> mgmt.SystemGetPropReq - 40, // 41: mgmt.MgmtSvc.FaultInjectReport:input_type -> chk.CheckReport - 41, // 42: mgmt.MgmtSvc.FaultInjectPoolFault:input_type -> chk.Fault - 41, // 43: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:input_type -> chk.Fault - 42, // 44: mgmt.MgmtSvc.Join:output_type -> mgmt.JoinResp - 43, // 45: mgmt.MgmtSvc.ClusterEvent:output_type -> shared.ClusterEventResp - 44, // 46: mgmt.MgmtSvc.LeaderQuery:output_type -> mgmt.LeaderQueryResp - 45, // 47: mgmt.MgmtSvc.PoolCreate:output_type -> mgmt.PoolCreateResp - 46, // 48: mgmt.MgmtSvc.PoolDestroy:output_type -> mgmt.PoolDestroyResp - 47, // 49: mgmt.MgmtSvc.PoolEvict:output_type -> mgmt.PoolEvictResp - 48, // 50: mgmt.MgmtSvc.PoolExclude:output_type -> mgmt.PoolExcludeResp - 49, // 51: mgmt.MgmtSvc.PoolDrain:output_type -> mgmt.PoolDrainResp - 50, // 52: mgmt.MgmtSvc.PoolExtend:output_type -> mgmt.PoolExtendResp - 51, // 53: mgmt.MgmtSvc.PoolReintegrate:output_type -> mgmt.PoolReintegrateResp - 52, // 54: mgmt.MgmtSvc.PoolQuery:output_type -> mgmt.PoolQueryResp - 53, // 55: mgmt.MgmtSvc.PoolQueryTarget:output_type -> mgmt.PoolQueryTargetResp - 54, // 56: mgmt.MgmtSvc.PoolSetProp:output_type -> mgmt.PoolSetPropResp - 55, // 57: mgmt.MgmtSvc.PoolGetProp:output_type -> mgmt.PoolGetPropResp - 56, // 58: mgmt.MgmtSvc.PoolGetACL:output_type -> mgmt.ACLResp - 56, // 59: mgmt.MgmtSvc.PoolOverwriteACL:output_type -> mgmt.ACLResp - 56, // 60: mgmt.MgmtSvc.PoolUpdateACL:output_type -> mgmt.ACLResp - 56, // 61: mgmt.MgmtSvc.PoolDeleteACL:output_type -> mgmt.ACLResp - 57, // 62: mgmt.MgmtSvc.GetAttachInfo:output_type -> mgmt.GetAttachInfoResp - 58, // 63: mgmt.MgmtSvc.ListPools:output_type -> mgmt.ListPoolsResp - 59, // 64: mgmt.MgmtSvc.ListContainers:output_type -> mgmt.ListContResp - 60, // 65: mgmt.MgmtSvc.ContSetOwner:output_type -> mgmt.DaosResp - 61, // 66: mgmt.MgmtSvc.SystemQuery:output_type -> mgmt.SystemQueryResp - 62, // 67: mgmt.MgmtSvc.SystemStop:output_type -> mgmt.SystemStopResp - 63, // 68: mgmt.MgmtSvc.SystemStart:output_type -> mgmt.SystemStartResp - 64, // 69: mgmt.MgmtSvc.SystemExclude:output_type -> mgmt.SystemExcludeResp - 65, // 70: mgmt.MgmtSvc.SystemErase:output_type -> mgmt.SystemEraseResp - 66, // 71: mgmt.MgmtSvc.SystemCleanup:output_type -> mgmt.SystemCleanupResp - 60, // 72: mgmt.MgmtSvc.SystemCheckEnable:output_type -> mgmt.DaosResp - 60, // 73: mgmt.MgmtSvc.SystemCheckDisable:output_type -> mgmt.DaosResp - 67, // 74: mgmt.MgmtSvc.SystemCheckStart:output_type -> mgmt.CheckStartResp - 68, // 75: mgmt.MgmtSvc.SystemCheckStop:output_type -> mgmt.CheckStopResp - 69, // 76: mgmt.MgmtSvc.SystemCheckQuery:output_type -> mgmt.CheckQueryResp - 60, // 77: mgmt.MgmtSvc.SystemCheckSetPolicy:output_type -> mgmt.DaosResp - 70, // 78: mgmt.MgmtSvc.SystemCheckGetPolicy:output_type -> mgmt.CheckGetPolicyResp - 71, // 79: mgmt.MgmtSvc.SystemCheckRepair:output_type -> mgmt.CheckActResp - 72, // 80: mgmt.MgmtSvc.PoolUpgrade:output_type -> mgmt.PoolUpgradeResp - 60, // 81: mgmt.MgmtSvc.SystemSetAttr:output_type -> mgmt.DaosResp - 73, // 82: mgmt.MgmtSvc.SystemGetAttr:output_type -> mgmt.SystemGetAttrResp - 60, // 83: mgmt.MgmtSvc.SystemSetProp:output_type -> mgmt.DaosResp - 74, // 84: mgmt.MgmtSvc.SystemGetProp:output_type -> mgmt.SystemGetPropResp - 60, // 85: mgmt.MgmtSvc.FaultInjectReport:output_type -> mgmt.DaosResp - 60, // 86: mgmt.MgmtSvc.FaultInjectPoolFault:output_type -> mgmt.DaosResp - 60, // 87: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:output_type -> mgmt.DaosResp - 44, // [44:88] is the sub-list for method output_type - 0, // [0:44] is the sub-list for method input_type + 25, // 26: mgmt.MgmtSvc.SystemDrain:input_type -> mgmt.SystemDrainReq + 26, // 27: mgmt.MgmtSvc.SystemErase:input_type -> mgmt.SystemEraseReq + 27, // 28: mgmt.MgmtSvc.SystemCleanup:input_type -> mgmt.SystemCleanupReq + 28, // 29: mgmt.MgmtSvc.SystemCheckEnable:input_type -> mgmt.CheckEnableReq + 29, // 30: mgmt.MgmtSvc.SystemCheckDisable:input_type -> mgmt.CheckDisableReq + 30, // 31: mgmt.MgmtSvc.SystemCheckStart:input_type -> mgmt.CheckStartReq + 31, // 32: mgmt.MgmtSvc.SystemCheckStop:input_type -> mgmt.CheckStopReq + 32, // 33: mgmt.MgmtSvc.SystemCheckQuery:input_type -> mgmt.CheckQueryReq + 33, // 34: mgmt.MgmtSvc.SystemCheckSetPolicy:input_type -> mgmt.CheckSetPolicyReq + 34, // 35: mgmt.MgmtSvc.SystemCheckGetPolicy:input_type -> mgmt.CheckGetPolicyReq + 35, // 36: mgmt.MgmtSvc.SystemCheckRepair:input_type -> mgmt.CheckActReq + 36, // 37: mgmt.MgmtSvc.PoolUpgrade:input_type -> mgmt.PoolUpgradeReq + 37, // 38: mgmt.MgmtSvc.SystemSetAttr:input_type -> mgmt.SystemSetAttrReq + 38, // 39: mgmt.MgmtSvc.SystemGetAttr:input_type -> mgmt.SystemGetAttrReq + 39, // 40: mgmt.MgmtSvc.SystemSetProp:input_type -> mgmt.SystemSetPropReq + 40, // 41: mgmt.MgmtSvc.SystemGetProp:input_type -> mgmt.SystemGetPropReq + 41, // 42: mgmt.MgmtSvc.FaultInjectReport:input_type -> chk.CheckReport + 42, // 43: mgmt.MgmtSvc.FaultInjectPoolFault:input_type -> chk.Fault + 42, // 44: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:input_type -> chk.Fault + 43, // 45: mgmt.MgmtSvc.Join:output_type -> mgmt.JoinResp + 44, // 46: mgmt.MgmtSvc.ClusterEvent:output_type -> shared.ClusterEventResp + 45, // 47: mgmt.MgmtSvc.LeaderQuery:output_type -> mgmt.LeaderQueryResp + 46, // 48: mgmt.MgmtSvc.PoolCreate:output_type -> mgmt.PoolCreateResp + 47, // 49: mgmt.MgmtSvc.PoolDestroy:output_type -> mgmt.PoolDestroyResp + 48, // 50: mgmt.MgmtSvc.PoolEvict:output_type -> mgmt.PoolEvictResp + 49, // 51: mgmt.MgmtSvc.PoolExclude:output_type -> mgmt.PoolExcludeResp + 50, // 52: mgmt.MgmtSvc.PoolDrain:output_type -> mgmt.PoolDrainResp + 51, // 53: mgmt.MgmtSvc.PoolExtend:output_type -> mgmt.PoolExtendResp + 52, // 54: mgmt.MgmtSvc.PoolReintegrate:output_type -> mgmt.PoolReintegrateResp + 53, // 55: mgmt.MgmtSvc.PoolQuery:output_type -> mgmt.PoolQueryResp + 54, // 56: mgmt.MgmtSvc.PoolQueryTarget:output_type -> mgmt.PoolQueryTargetResp + 55, // 57: mgmt.MgmtSvc.PoolSetProp:output_type -> mgmt.PoolSetPropResp + 56, // 58: mgmt.MgmtSvc.PoolGetProp:output_type -> mgmt.PoolGetPropResp + 57, // 59: mgmt.MgmtSvc.PoolGetACL:output_type -> mgmt.ACLResp + 57, // 60: mgmt.MgmtSvc.PoolOverwriteACL:output_type -> mgmt.ACLResp + 57, // 61: mgmt.MgmtSvc.PoolUpdateACL:output_type -> mgmt.ACLResp + 57, // 62: mgmt.MgmtSvc.PoolDeleteACL:output_type -> mgmt.ACLResp + 58, // 63: mgmt.MgmtSvc.GetAttachInfo:output_type -> mgmt.GetAttachInfoResp + 59, // 64: mgmt.MgmtSvc.ListPools:output_type -> mgmt.ListPoolsResp + 60, // 65: mgmt.MgmtSvc.ListContainers:output_type -> mgmt.ListContResp + 61, // 66: mgmt.MgmtSvc.ContSetOwner:output_type -> mgmt.DaosResp + 62, // 67: mgmt.MgmtSvc.SystemQuery:output_type -> mgmt.SystemQueryResp + 63, // 68: mgmt.MgmtSvc.SystemStop:output_type -> mgmt.SystemStopResp + 64, // 69: mgmt.MgmtSvc.SystemStart:output_type -> mgmt.SystemStartResp + 65, // 70: mgmt.MgmtSvc.SystemExclude:output_type -> mgmt.SystemExcludeResp + 66, // 71: mgmt.MgmtSvc.SystemDrain:output_type -> mgmt.SystemDrainResp + 67, // 72: mgmt.MgmtSvc.SystemErase:output_type -> mgmt.SystemEraseResp + 68, // 73: mgmt.MgmtSvc.SystemCleanup:output_type -> mgmt.SystemCleanupResp + 61, // 74: mgmt.MgmtSvc.SystemCheckEnable:output_type -> mgmt.DaosResp + 61, // 75: mgmt.MgmtSvc.SystemCheckDisable:output_type -> mgmt.DaosResp + 69, // 76: mgmt.MgmtSvc.SystemCheckStart:output_type -> mgmt.CheckStartResp + 70, // 77: mgmt.MgmtSvc.SystemCheckStop:output_type -> mgmt.CheckStopResp + 71, // 78: mgmt.MgmtSvc.SystemCheckQuery:output_type -> mgmt.CheckQueryResp + 61, // 79: mgmt.MgmtSvc.SystemCheckSetPolicy:output_type -> mgmt.DaosResp + 72, // 80: mgmt.MgmtSvc.SystemCheckGetPolicy:output_type -> mgmt.CheckGetPolicyResp + 73, // 81: mgmt.MgmtSvc.SystemCheckRepair:output_type -> mgmt.CheckActResp + 74, // 82: mgmt.MgmtSvc.PoolUpgrade:output_type -> mgmt.PoolUpgradeResp + 61, // 83: mgmt.MgmtSvc.SystemSetAttr:output_type -> mgmt.DaosResp + 75, // 84: mgmt.MgmtSvc.SystemGetAttr:output_type -> mgmt.SystemGetAttrResp + 61, // 85: mgmt.MgmtSvc.SystemSetProp:output_type -> mgmt.DaosResp + 76, // 86: mgmt.MgmtSvc.SystemGetProp:output_type -> mgmt.SystemGetPropResp + 61, // 87: mgmt.MgmtSvc.FaultInjectReport:output_type -> mgmt.DaosResp + 61, // 88: mgmt.MgmtSvc.FaultInjectPoolFault:output_type -> mgmt.DaosResp + 61, // 89: mgmt.MgmtSvc.FaultInjectMgmtPoolFault:output_type -> mgmt.DaosResp + 45, // [45:90] is the sub-list for method output_type + 0, // [0:45] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/src/control/common/proto/mgmt/mgmt_grpc.pb.go b/src/control/common/proto/mgmt/mgmt_grpc.pb.go index bf69683f90c..dc9e2b1555a 100644 --- a/src/control/common/proto/mgmt/mgmt_grpc.pb.go +++ b/src/control/common/proto/mgmt/mgmt_grpc.pb.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -53,6 +53,7 @@ const ( MgmtSvc_SystemStop_FullMethodName = "/mgmt.MgmtSvc/SystemStop" MgmtSvc_SystemStart_FullMethodName = "/mgmt.MgmtSvc/SystemStart" MgmtSvc_SystemExclude_FullMethodName = "/mgmt.MgmtSvc/SystemExclude" + MgmtSvc_SystemDrain_FullMethodName = "/mgmt.MgmtSvc/SystemDrain" MgmtSvc_SystemErase_FullMethodName = "/mgmt.MgmtSvc/SystemErase" MgmtSvc_SystemCleanup_FullMethodName = "/mgmt.MgmtSvc/SystemCleanup" MgmtSvc_SystemCheckEnable_FullMethodName = "/mgmt.MgmtSvc/SystemCheckEnable" @@ -130,6 +131,8 @@ type MgmtSvcClient interface { SystemStart(ctx context.Context, in *SystemStartReq, opts ...grpc.CallOption) (*SystemStartResp, error) // Exclude DAOS ranks SystemExclude(ctx context.Context, in *SystemExcludeReq, opts ...grpc.CallOption) (*SystemExcludeResp, error) + // Drain DAOS ranks from all pools + SystemDrain(ctx context.Context, in *SystemDrainReq, opts ...grpc.CallOption) (*SystemDrainResp, error) // Erase DAOS system database prior to reformat SystemErase(ctx context.Context, in *SystemEraseReq, opts ...grpc.CallOption) (*SystemEraseResp, error) // Clean up leaked resources for a given node @@ -410,6 +413,15 @@ func (c *mgmtSvcClient) SystemExclude(ctx context.Context, in *SystemExcludeReq, return out, nil } +func (c *mgmtSvcClient) SystemDrain(ctx context.Context, in *SystemDrainReq, opts ...grpc.CallOption) (*SystemDrainResp, error) { + out := new(SystemDrainResp) + err := c.cc.Invoke(ctx, MgmtSvc_SystemDrain_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *mgmtSvcClient) SystemErase(ctx context.Context, in *SystemEraseReq, opts ...grpc.CallOption) (*SystemEraseResp, error) { out := new(SystemEraseResp) err := c.cc.Invoke(ctx, MgmtSvc_SystemErase_FullMethodName, in, out, opts...) @@ -629,6 +641,8 @@ type MgmtSvcServer interface { SystemStart(context.Context, *SystemStartReq) (*SystemStartResp, error) // Exclude DAOS ranks SystemExclude(context.Context, *SystemExcludeReq) (*SystemExcludeResp, error) + // Drain DAOS ranks from all pools + SystemDrain(context.Context, *SystemDrainReq) (*SystemDrainResp, error) // Erase DAOS system database prior to reformat SystemErase(context.Context, *SystemEraseReq) (*SystemEraseResp, error) // Clean up leaked resources for a given node @@ -750,6 +764,9 @@ func (UnimplementedMgmtSvcServer) SystemStart(context.Context, *SystemStartReq) func (UnimplementedMgmtSvcServer) SystemExclude(context.Context, *SystemExcludeReq) (*SystemExcludeResp, error) { return nil, status.Errorf(codes.Unimplemented, "method SystemExclude not implemented") } +func (UnimplementedMgmtSvcServer) SystemDrain(context.Context, *SystemDrainReq) (*SystemDrainResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method SystemDrain not implemented") +} func (UnimplementedMgmtSvcServer) SystemErase(context.Context, *SystemEraseReq) (*SystemEraseResp, error) { return nil, status.Errorf(codes.Unimplemented, "method SystemErase not implemented") } @@ -1285,6 +1302,24 @@ func _MgmtSvc_SystemExclude_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } +func _MgmtSvc_SystemDrain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SystemDrainReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MgmtSvcServer).SystemDrain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MgmtSvc_SystemDrain_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MgmtSvcServer).SystemDrain(ctx, req.(*SystemDrainReq)) + } + return interceptor(ctx, in, info, handler) +} + func _MgmtSvc_SystemErase_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SystemEraseReq) if err := dec(in); err != nil { @@ -1720,6 +1755,10 @@ var MgmtSvc_ServiceDesc = grpc.ServiceDesc{ MethodName: "SystemExclude", Handler: _MgmtSvc_SystemExclude_Handler, }, + { + MethodName: "SystemDrain", + Handler: _MgmtSvc_SystemDrain_Handler, + }, { MethodName: "SystemErase", Handler: _MgmtSvc_SystemErase_Handler, diff --git a/src/control/common/proto/mgmt/system.pb.go b/src/control/common/proto/mgmt/system.pb.go index b4d66dd7364..afddaa4fc22 100644 --- a/src/control/common/proto/mgmt/system.pb.go +++ b/src/control/common/proto/mgmt/system.pb.go @@ -7,7 +7,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v3.21.12 +// protoc v3.5.0 // source: mgmt/system.proto package mgmt @@ -567,6 +567,118 @@ func (x *SystemExcludeResp) GetResults() []*shared.RankResult { return nil } +// SystemDrainReq supplies system-drain parameters. +type SystemDrainReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Sys string `protobuf:"bytes,1,opt,name=sys,proto3" json:"sys,omitempty"` // DAOS system name + Ranks string `protobuf:"bytes,2,opt,name=ranks,proto3" json:"ranks,omitempty"` // rankset to drain on all pools + Hosts string `protobuf:"bytes,3,opt,name=hosts,proto3" json:"hosts,omitempty"` // hostset to drain on all pools +} + +func (x *SystemDrainReq) Reset() { + *x = SystemDrainReq{} + if protoimpl.UnsafeEnabled { + mi := &file_mgmt_system_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SystemDrainReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SystemDrainReq) ProtoMessage() {} + +func (x *SystemDrainReq) ProtoReflect() protoreflect.Message { + mi := &file_mgmt_system_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SystemDrainReq.ProtoReflect.Descriptor instead. +func (*SystemDrainReq) Descriptor() ([]byte, []int) { + return file_mgmt_system_proto_rawDescGZIP(), []int{7} +} + +func (x *SystemDrainReq) GetSys() string { + if x != nil { + return x.Sys + } + return "" +} + +func (x *SystemDrainReq) GetRanks() string { + if x != nil { + return x.Ranks + } + return "" +} + +func (x *SystemDrainReq) GetHosts() string { + if x != nil { + return x.Hosts + } + return "" +} + +// SystemDrainResp returns status of system-drain request. +type SystemDrainResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Results []*SystemDrainResp_DrainResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Results for individual pool-ranks drain calls. +} + +func (x *SystemDrainResp) Reset() { + *x = SystemDrainResp{} + if protoimpl.UnsafeEnabled { + mi := &file_mgmt_system_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SystemDrainResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SystemDrainResp) ProtoMessage() {} + +func (x *SystemDrainResp) ProtoReflect() protoreflect.Message { + mi := &file_mgmt_system_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SystemDrainResp.ProtoReflect.Descriptor instead. +func (*SystemDrainResp) Descriptor() ([]byte, []int) { + return file_mgmt_system_proto_rawDescGZIP(), []int{8} +} + +func (x *SystemDrainResp) GetResults() []*SystemDrainResp_DrainResult { + if x != nil { + return x.Results + } + return nil +} + // SystemQueryReq supplies system query parameters. type SystemQueryReq struct { state protoimpl.MessageState @@ -582,7 +694,7 @@ type SystemQueryReq struct { func (x *SystemQueryReq) Reset() { *x = SystemQueryReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[7] + mi := &file_mgmt_system_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -595,7 +707,7 @@ func (x *SystemQueryReq) String() string { func (*SystemQueryReq) ProtoMessage() {} func (x *SystemQueryReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[7] + mi := &file_mgmt_system_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -608,7 +720,7 @@ func (x *SystemQueryReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemQueryReq.ProtoReflect.Descriptor instead. func (*SystemQueryReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{7} + return file_mgmt_system_proto_rawDescGZIP(), []int{9} } func (x *SystemQueryReq) GetSys() string { @@ -655,7 +767,7 @@ type SystemQueryResp struct { func (x *SystemQueryResp) Reset() { *x = SystemQueryResp{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[8] + mi := &file_mgmt_system_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -668,7 +780,7 @@ func (x *SystemQueryResp) String() string { func (*SystemQueryResp) ProtoMessage() {} func (x *SystemQueryResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[8] + mi := &file_mgmt_system_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -681,7 +793,7 @@ func (x *SystemQueryResp) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemQueryResp.ProtoReflect.Descriptor instead. func (*SystemQueryResp) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{8} + return file_mgmt_system_proto_rawDescGZIP(), []int{10} } func (x *SystemQueryResp) GetMembers() []*SystemMember { @@ -731,7 +843,7 @@ type SystemEraseReq struct { func (x *SystemEraseReq) Reset() { *x = SystemEraseReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[9] + mi := &file_mgmt_system_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -744,7 +856,7 @@ func (x *SystemEraseReq) String() string { func (*SystemEraseReq) ProtoMessage() {} func (x *SystemEraseReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[9] + mi := &file_mgmt_system_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -757,7 +869,7 @@ func (x *SystemEraseReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemEraseReq.ProtoReflect.Descriptor instead. func (*SystemEraseReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{9} + return file_mgmt_system_proto_rawDescGZIP(), []int{11} } func (x *SystemEraseReq) GetSys() string { @@ -778,7 +890,7 @@ type SystemEraseResp struct { func (x *SystemEraseResp) Reset() { *x = SystemEraseResp{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[10] + mi := &file_mgmt_system_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -791,7 +903,7 @@ func (x *SystemEraseResp) String() string { func (*SystemEraseResp) ProtoMessage() {} func (x *SystemEraseResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[10] + mi := &file_mgmt_system_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -804,7 +916,7 @@ func (x *SystemEraseResp) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemEraseResp.ProtoReflect.Descriptor instead. func (*SystemEraseResp) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{10} + return file_mgmt_system_proto_rawDescGZIP(), []int{12} } func (x *SystemEraseResp) GetResults() []*shared.RankResult { @@ -827,7 +939,7 @@ type SystemCleanupReq struct { func (x *SystemCleanupReq) Reset() { *x = SystemCleanupReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[11] + mi := &file_mgmt_system_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -840,7 +952,7 @@ func (x *SystemCleanupReq) String() string { func (*SystemCleanupReq) ProtoMessage() {} func (x *SystemCleanupReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[11] + mi := &file_mgmt_system_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -853,7 +965,7 @@ func (x *SystemCleanupReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemCleanupReq.ProtoReflect.Descriptor instead. func (*SystemCleanupReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{11} + return file_mgmt_system_proto_rawDescGZIP(), []int{13} } func (x *SystemCleanupReq) GetSys() string { @@ -882,7 +994,7 @@ type SystemCleanupResp struct { func (x *SystemCleanupResp) Reset() { *x = SystemCleanupResp{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[12] + mi := &file_mgmt_system_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -895,7 +1007,7 @@ func (x *SystemCleanupResp) String() string { func (*SystemCleanupResp) ProtoMessage() {} func (x *SystemCleanupResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[12] + mi := &file_mgmt_system_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -908,7 +1020,7 @@ func (x *SystemCleanupResp) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemCleanupResp.ProtoReflect.Descriptor instead. func (*SystemCleanupResp) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{12} + return file_mgmt_system_proto_rawDescGZIP(), []int{14} } func (x *SystemCleanupResp) GetResults() []*SystemCleanupResp_CleanupResult { @@ -931,7 +1043,7 @@ type SystemSetAttrReq struct { func (x *SystemSetAttrReq) Reset() { *x = SystemSetAttrReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[13] + mi := &file_mgmt_system_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -944,7 +1056,7 @@ func (x *SystemSetAttrReq) String() string { func (*SystemSetAttrReq) ProtoMessage() {} func (x *SystemSetAttrReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[13] + mi := &file_mgmt_system_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -957,7 +1069,7 @@ func (x *SystemSetAttrReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemSetAttrReq.ProtoReflect.Descriptor instead. func (*SystemSetAttrReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{13} + return file_mgmt_system_proto_rawDescGZIP(), []int{15} } func (x *SystemSetAttrReq) GetSys() string { @@ -988,7 +1100,7 @@ type SystemGetAttrReq struct { func (x *SystemGetAttrReq) Reset() { *x = SystemGetAttrReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[14] + mi := &file_mgmt_system_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1001,7 +1113,7 @@ func (x *SystemGetAttrReq) String() string { func (*SystemGetAttrReq) ProtoMessage() {} func (x *SystemGetAttrReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[14] + mi := &file_mgmt_system_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1014,7 +1126,7 @@ func (x *SystemGetAttrReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemGetAttrReq.ProtoReflect.Descriptor instead. func (*SystemGetAttrReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{14} + return file_mgmt_system_proto_rawDescGZIP(), []int{16} } func (x *SystemGetAttrReq) GetSys() string { @@ -1043,7 +1155,7 @@ type SystemGetAttrResp struct { func (x *SystemGetAttrResp) Reset() { *x = SystemGetAttrResp{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[15] + mi := &file_mgmt_system_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1056,7 +1168,7 @@ func (x *SystemGetAttrResp) String() string { func (*SystemGetAttrResp) ProtoMessage() {} func (x *SystemGetAttrResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[15] + mi := &file_mgmt_system_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1069,7 +1181,7 @@ func (x *SystemGetAttrResp) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemGetAttrResp.ProtoReflect.Descriptor instead. func (*SystemGetAttrResp) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{15} + return file_mgmt_system_proto_rawDescGZIP(), []int{17} } func (x *SystemGetAttrResp) GetAttributes() map[string]string { @@ -1092,7 +1204,7 @@ type SystemSetPropReq struct { func (x *SystemSetPropReq) Reset() { *x = SystemSetPropReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[16] + mi := &file_mgmt_system_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1105,7 +1217,7 @@ func (x *SystemSetPropReq) String() string { func (*SystemSetPropReq) ProtoMessage() {} func (x *SystemSetPropReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[16] + mi := &file_mgmt_system_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1118,7 +1230,7 @@ func (x *SystemSetPropReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemSetPropReq.ProtoReflect.Descriptor instead. func (*SystemSetPropReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{16} + return file_mgmt_system_proto_rawDescGZIP(), []int{18} } func (x *SystemSetPropReq) GetSys() string { @@ -1149,7 +1261,7 @@ type SystemGetPropReq struct { func (x *SystemGetPropReq) Reset() { *x = SystemGetPropReq{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[17] + mi := &file_mgmt_system_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1162,7 +1274,7 @@ func (x *SystemGetPropReq) String() string { func (*SystemGetPropReq) ProtoMessage() {} func (x *SystemGetPropReq) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[17] + mi := &file_mgmt_system_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1175,7 +1287,7 @@ func (x *SystemGetPropReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemGetPropReq.ProtoReflect.Descriptor instead. func (*SystemGetPropReq) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{17} + return file_mgmt_system_proto_rawDescGZIP(), []int{19} } func (x *SystemGetPropReq) GetSys() string { @@ -1204,7 +1316,7 @@ type SystemGetPropResp struct { func (x *SystemGetPropResp) Reset() { *x = SystemGetPropResp{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[18] + mi := &file_mgmt_system_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1217,7 +1329,7 @@ func (x *SystemGetPropResp) String() string { func (*SystemGetPropResp) ProtoMessage() {} func (x *SystemGetPropResp) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[18] + mi := &file_mgmt_system_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1230,7 +1342,7 @@ func (x *SystemGetPropResp) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemGetPropResp.ProtoReflect.Descriptor instead. func (*SystemGetPropResp) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{18} + return file_mgmt_system_proto_rawDescGZIP(), []int{20} } func (x *SystemGetPropResp) GetProperties() map[string]string { @@ -1240,6 +1352,77 @@ func (x *SystemGetPropResp) GetProperties() map[string]string { return nil } +type SystemDrainResp_DrainResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Status int32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // Status of the evict on the specific pool + Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` // Error message if status indicates an error + PoolId string `protobuf:"bytes,3,opt,name=pool_id,json=poolId,proto3" json:"pool_id,omitempty"` // uuid of pool + Ranks string `protobuf:"bytes,4,opt,name=ranks,proto3" json:"ranks,omitempty"` // rankset to have drained on poolexclude +} + +func (x *SystemDrainResp_DrainResult) Reset() { + *x = SystemDrainResp_DrainResult{} + if protoimpl.UnsafeEnabled { + mi := &file_mgmt_system_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SystemDrainResp_DrainResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SystemDrainResp_DrainResult) ProtoMessage() {} + +func (x *SystemDrainResp_DrainResult) ProtoReflect() protoreflect.Message { + mi := &file_mgmt_system_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SystemDrainResp_DrainResult.ProtoReflect.Descriptor instead. +func (*SystemDrainResp_DrainResult) Descriptor() ([]byte, []int) { + return file_mgmt_system_proto_rawDescGZIP(), []int{8, 0} +} + +func (x *SystemDrainResp_DrainResult) GetStatus() int32 { + if x != nil { + return x.Status + } + return 0 +} + +func (x *SystemDrainResp_DrainResult) GetMsg() string { + if x != nil { + return x.Msg + } + return "" +} + +func (x *SystemDrainResp_DrainResult) GetPoolId() string { + if x != nil { + return x.PoolId + } + return "" +} + +func (x *SystemDrainResp_DrainResult) GetRanks() string { + if x != nil { + return x.Ranks + } + return "" +} + type SystemCleanupResp_CleanupResult struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1254,7 +1437,7 @@ type SystemCleanupResp_CleanupResult struct { func (x *SystemCleanupResp_CleanupResult) Reset() { *x = SystemCleanupResp_CleanupResult{} if protoimpl.UnsafeEnabled { - mi := &file_mgmt_system_proto_msgTypes[19] + mi := &file_mgmt_system_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1267,7 +1450,7 @@ func (x *SystemCleanupResp_CleanupResult) String() string { func (*SystemCleanupResp_CleanupResult) ProtoMessage() {} func (x *SystemCleanupResp_CleanupResult) ProtoReflect() protoreflect.Message { - mi := &file_mgmt_system_proto_msgTypes[19] + mi := &file_mgmt_system_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1280,7 +1463,7 @@ func (x *SystemCleanupResp_CleanupResult) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemCleanupResp_CleanupResult.ProtoReflect.Descriptor instead. func (*SystemCleanupResp_CleanupResult) Descriptor() ([]byte, []int) { - return file_mgmt_system_proto_rawDescGZIP(), []int{12, 0} + return file_mgmt_system_proto_rawDescGZIP(), []int{14, 0} } func (x *SystemCleanupResp_CleanupResult) GetStatus() int32 { @@ -1381,102 +1564,118 @@ var file_mgmt_system_proto_rawDesc = []byte{ 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2c, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x52, 0x61, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x6d, 0x0a, 0x0e, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, + 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x4e, 0x0a, 0x0e, 0x53, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, 0x6b, 0x22, 0xc4, 0x01, 0x0a, 0x0f, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2c, 0x0a, 0x07, - 0x6d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x6d, 0x62, 0x65, - 0x72, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x61, 0x62, - 0x73, 0x65, 0x6e, 0x74, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, - 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x21, - 0x0a, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x22, - 0x22, 0x0a, 0x0e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, + 0x28, 0x09, 0x52, 0x05, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x0f, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x12, 0x3b, 0x0a, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x72, 0x61, 0x69, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x66, 0x0a, 0x0b, 0x44, 0x72, + 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6d, 0x73, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6f, 0x6f, 0x6c, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x61, 0x6e, + 0x6b, 0x73, 0x22, 0x6d, 0x0a, 0x0e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x14, 0x0a, 0x05, + 0x68, 0x6f, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x68, 0x6f, 0x73, + 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x73, 0x6b, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, + 0x6b, 0x22, 0xc4, 0x01, 0x0a, 0x0f, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2c, 0x0a, 0x07, 0x6d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x62, + 0x65, 0x72, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, 0x72, 0x61, 0x6e, + 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, + 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x61, 0x62, 0x73, 0x65, 0x6e, 0x74, 0x68, + 0x6f, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x62, 0x73, 0x65, + 0x6e, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x64, + 0x61, 0x74, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x70, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x22, 0x22, 0x0a, 0x0e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x22, 0x3f, 0x0a, 0x0f, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x2c, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x52, 0x61, 0x6e, 0x6b, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x3e, 0x0a, + 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x73, 0x79, 0x73, 0x22, 0x3f, 0x0a, 0x0f, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x72, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2c, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, - 0x2e, 0x52, 0x61, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x22, 0x3e, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, - 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, - 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, - 0x52, 0x65, 0x73, 0x70, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x68, 0x0a, 0x0d, 0x43, - 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6f, 0x6f, 0x6c, 0x49, 0x64, 0x12, - 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xab, 0x01, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x46, 0x0a, 0x0a, - 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x26, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, - 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, - 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, - 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x38, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, - 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x9b, 0x01, - 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, - 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x01, 0x0a, 0x10, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, + 0x73, 0x79, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0xbe, 0x01, + 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x70, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x2e, 0x43, 0x6c, + 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x68, 0x0a, 0x0d, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, + 0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, + 0x17, 0x0a, 0x07, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x70, 0x6f, 0x6f, 0x6c, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xab, + 0x01, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, + 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x46, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, + 0x71, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, + 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x38, 0x0a, 0x10, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, - 0x79, 0x73, 0x12, 0x46, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x2e, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x38, 0x0a, 0x10, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, - 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, - 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, - 0x65, 0x79, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, - 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x47, 0x0a, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, - 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, - 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x79, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x47, 0x65, 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x47, 0x0a, 0x0a, + 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, + 0x74, 0x41, 0x74, 0x74, 0x72, 0x52, 0x65, 0x73, 0x70, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, + 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x01, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, + 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x46, 0x0a, 0x0a, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x26, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x38, 0x0a, 0x10, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, + 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x9b, 0x01, 0x0a, + 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, + 0x73, 0x70, 0x12, 0x47, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x2e, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, + 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1491,7 +1690,7 @@ func file_mgmt_system_proto_rawDescGZIP() []byte { return file_mgmt_system_proto_rawDescData } -var file_mgmt_system_proto_msgTypes = make([]protoimpl.MessageInfo, 24) +var file_mgmt_system_proto_msgTypes = make([]protoimpl.MessageInfo, 27) var file_mgmt_system_proto_goTypes = []interface{}{ (*SystemMember)(nil), // 0: mgmt.SystemMember (*SystemStopReq)(nil), // 1: mgmt.SystemStopReq @@ -1500,41 +1699,45 @@ var file_mgmt_system_proto_goTypes = []interface{}{ (*SystemStartResp)(nil), // 4: mgmt.SystemStartResp (*SystemExcludeReq)(nil), // 5: mgmt.SystemExcludeReq (*SystemExcludeResp)(nil), // 6: mgmt.SystemExcludeResp - (*SystemQueryReq)(nil), // 7: mgmt.SystemQueryReq - (*SystemQueryResp)(nil), // 8: mgmt.SystemQueryResp - (*SystemEraseReq)(nil), // 9: mgmt.SystemEraseReq - (*SystemEraseResp)(nil), // 10: mgmt.SystemEraseResp - (*SystemCleanupReq)(nil), // 11: mgmt.SystemCleanupReq - (*SystemCleanupResp)(nil), // 12: mgmt.SystemCleanupResp - (*SystemSetAttrReq)(nil), // 13: mgmt.SystemSetAttrReq - (*SystemGetAttrReq)(nil), // 14: mgmt.SystemGetAttrReq - (*SystemGetAttrResp)(nil), // 15: mgmt.SystemGetAttrResp - (*SystemSetPropReq)(nil), // 16: mgmt.SystemSetPropReq - (*SystemGetPropReq)(nil), // 17: mgmt.SystemGetPropReq - (*SystemGetPropResp)(nil), // 18: mgmt.SystemGetPropResp - (*SystemCleanupResp_CleanupResult)(nil), // 19: mgmt.SystemCleanupResp.CleanupResult - nil, // 20: mgmt.SystemSetAttrReq.AttributesEntry - nil, // 21: mgmt.SystemGetAttrResp.AttributesEntry - nil, // 22: mgmt.SystemSetPropReq.PropertiesEntry - nil, // 23: mgmt.SystemGetPropResp.PropertiesEntry - (*shared.RankResult)(nil), // 24: shared.RankResult + (*SystemDrainReq)(nil), // 7: mgmt.SystemDrainReq + (*SystemDrainResp)(nil), // 8: mgmt.SystemDrainResp + (*SystemQueryReq)(nil), // 9: mgmt.SystemQueryReq + (*SystemQueryResp)(nil), // 10: mgmt.SystemQueryResp + (*SystemEraseReq)(nil), // 11: mgmt.SystemEraseReq + (*SystemEraseResp)(nil), // 12: mgmt.SystemEraseResp + (*SystemCleanupReq)(nil), // 13: mgmt.SystemCleanupReq + (*SystemCleanupResp)(nil), // 14: mgmt.SystemCleanupResp + (*SystemSetAttrReq)(nil), // 15: mgmt.SystemSetAttrReq + (*SystemGetAttrReq)(nil), // 16: mgmt.SystemGetAttrReq + (*SystemGetAttrResp)(nil), // 17: mgmt.SystemGetAttrResp + (*SystemSetPropReq)(nil), // 18: mgmt.SystemSetPropReq + (*SystemGetPropReq)(nil), // 19: mgmt.SystemGetPropReq + (*SystemGetPropResp)(nil), // 20: mgmt.SystemGetPropResp + (*SystemDrainResp_DrainResult)(nil), // 21: mgmt.SystemDrainResp.DrainResult + (*SystemCleanupResp_CleanupResult)(nil), // 22: mgmt.SystemCleanupResp.CleanupResult + nil, // 23: mgmt.SystemSetAttrReq.AttributesEntry + nil, // 24: mgmt.SystemGetAttrResp.AttributesEntry + nil, // 25: mgmt.SystemSetPropReq.PropertiesEntry + nil, // 26: mgmt.SystemGetPropResp.PropertiesEntry + (*shared.RankResult)(nil), // 27: shared.RankResult } var file_mgmt_system_proto_depIdxs = []int32{ - 24, // 0: mgmt.SystemStopResp.results:type_name -> shared.RankResult - 24, // 1: mgmt.SystemStartResp.results:type_name -> shared.RankResult - 24, // 2: mgmt.SystemExcludeResp.results:type_name -> shared.RankResult - 0, // 3: mgmt.SystemQueryResp.members:type_name -> mgmt.SystemMember - 24, // 4: mgmt.SystemEraseResp.results:type_name -> shared.RankResult - 19, // 5: mgmt.SystemCleanupResp.results:type_name -> mgmt.SystemCleanupResp.CleanupResult - 20, // 6: mgmt.SystemSetAttrReq.attributes:type_name -> mgmt.SystemSetAttrReq.AttributesEntry - 21, // 7: mgmt.SystemGetAttrResp.attributes:type_name -> mgmt.SystemGetAttrResp.AttributesEntry - 22, // 8: mgmt.SystemSetPropReq.properties:type_name -> mgmt.SystemSetPropReq.PropertiesEntry - 23, // 9: mgmt.SystemGetPropResp.properties:type_name -> mgmt.SystemGetPropResp.PropertiesEntry - 10, // [10:10] is the sub-list for method output_type - 10, // [10:10] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 27, // 0: mgmt.SystemStopResp.results:type_name -> shared.RankResult + 27, // 1: mgmt.SystemStartResp.results:type_name -> shared.RankResult + 27, // 2: mgmt.SystemExcludeResp.results:type_name -> shared.RankResult + 21, // 3: mgmt.SystemDrainResp.results:type_name -> mgmt.SystemDrainResp.DrainResult + 0, // 4: mgmt.SystemQueryResp.members:type_name -> mgmt.SystemMember + 27, // 5: mgmt.SystemEraseResp.results:type_name -> shared.RankResult + 22, // 6: mgmt.SystemCleanupResp.results:type_name -> mgmt.SystemCleanupResp.CleanupResult + 23, // 7: mgmt.SystemSetAttrReq.attributes:type_name -> mgmt.SystemSetAttrReq.AttributesEntry + 24, // 8: mgmt.SystemGetAttrResp.attributes:type_name -> mgmt.SystemGetAttrResp.AttributesEntry + 25, // 9: mgmt.SystemSetPropReq.properties:type_name -> mgmt.SystemSetPropReq.PropertiesEntry + 26, // 10: mgmt.SystemGetPropResp.properties:type_name -> mgmt.SystemGetPropResp.PropertiesEntry + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_mgmt_system_proto_init() } @@ -1628,7 +1831,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemQueryReq); i { + switch v := v.(*SystemDrainReq); i { case 0: return &v.state case 1: @@ -1640,7 +1843,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemQueryResp); i { + switch v := v.(*SystemDrainResp); i { case 0: return &v.state case 1: @@ -1652,7 +1855,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemEraseReq); i { + switch v := v.(*SystemQueryReq); i { case 0: return &v.state case 1: @@ -1664,7 +1867,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemEraseResp); i { + switch v := v.(*SystemQueryResp); i { case 0: return &v.state case 1: @@ -1676,7 +1879,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemCleanupReq); i { + switch v := v.(*SystemEraseReq); i { case 0: return &v.state case 1: @@ -1688,7 +1891,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemCleanupResp); i { + switch v := v.(*SystemEraseResp); i { case 0: return &v.state case 1: @@ -1700,7 +1903,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemSetAttrReq); i { + switch v := v.(*SystemCleanupReq); i { case 0: return &v.state case 1: @@ -1712,7 +1915,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemGetAttrReq); i { + switch v := v.(*SystemCleanupResp); i { case 0: return &v.state case 1: @@ -1724,7 +1927,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemGetAttrResp); i { + switch v := v.(*SystemSetAttrReq); i { case 0: return &v.state case 1: @@ -1736,7 +1939,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemSetPropReq); i { + switch v := v.(*SystemGetAttrReq); i { case 0: return &v.state case 1: @@ -1748,7 +1951,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemGetPropReq); i { + switch v := v.(*SystemGetAttrResp); i { case 0: return &v.state case 1: @@ -1760,7 +1963,7 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemGetPropResp); i { + switch v := v.(*SystemSetPropReq); i { case 0: return &v.state case 1: @@ -1772,6 +1975,42 @@ func file_mgmt_system_proto_init() { } } file_mgmt_system_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SystemGetPropReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mgmt_system_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SystemGetPropResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mgmt_system_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SystemDrainResp_DrainResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mgmt_system_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SystemCleanupResp_CleanupResult); i { case 0: return &v.state @@ -1790,7 +2029,7 @@ func file_mgmt_system_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_mgmt_system_proto_rawDesc, NumEnums: 0, - NumMessages: 24, + NumMessages: 27, NumExtensions: 0, NumServices: 0, }, diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go index 7b17d865029..c5f3edac531 100644 --- a/src/control/lib/control/pool.go +++ b/src/control/lib/control/pool.go @@ -533,7 +533,7 @@ func poolQueryInt(ctx context.Context, rpcClient UnaryInvoker, req *PoolQueryReq return resp, err } -// UpdateState update the pool state based on response field values. +// UpdateState updates the pool state based on response field values. func (pqr *PoolQueryResp) UpdateState() error { // Update the state as Ready if DAOS return code is 0. if pqr.Status == 0 { @@ -785,9 +785,8 @@ type PoolDrainReq struct { // DrainResp has no other parameters other than success/failure for now. -// PoolDrain will set a pool target for a specific rank to the drain status. -// This should automatically start the rebuildiing process. -// Returns an error (including any DER code from DAOS). +// PoolDrain will set a pool target for a specific rank in to the drain state which should +// automatically start the rebuildiing process. Returns an error (including any DER code from DAOS). func PoolDrain(ctx context.Context, rpcClient UnaryInvoker, req *PoolDrainReq) error { pbReq := &mgmtpb.PoolDrainReq{ Sys: req.getSystem(rpcClient), diff --git a/src/control/lib/control/response_test.go b/src/control/lib/control/response_test.go index d9838ea6045..61d3bbb1760 100644 --- a/src/control/lib/control/response_test.go +++ b/src/control/lib/control/response_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2022 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -18,6 +18,7 @@ import ( mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/lib/hostlist" + "github.com/daos-stack/daos/src/control/lib/ranklist" ) func defResCmpOpts() []cmp.Option { @@ -25,6 +26,9 @@ func defResCmpOpts() []cmp.Option { cmp.Comparer(func(x, y *hostlist.HostSet) bool { return x.RangedString() == y.RangedString() }), + cmp.Comparer(func(x, y *ranklist.RankSet) bool { + return x.RangedString() == y.RangedString() + }), cmpopts.IgnoreFields(HostErrorSet{}, "HostError"), } } diff --git a/src/control/lib/control/system.go b/src/control/lib/control/system.go index 73afa873c9f..876c2ed1eee 100644 --- a/src/control/lib/control/system.go +++ b/src/control/lib/control/system.go @@ -68,11 +68,11 @@ func (req *sysRequest) SetHosts(hosts *hostlist.HostSet) { } type sysResponse struct { - AbsentRanks ranklist.RankSet `json:"-"` - AbsentHosts hostlist.HostSet `json:"-"` + AbsentRanks ranklist.RankSet + AbsentHosts hostlist.HostSet } -func (resp *sysResponse) getAbsentHostsRanks(inHosts, inRanks string) error { +func (resp *sysResponse) setAbsentHostsRanks(inHosts, inRanks string) error { ahs, err := hostlist.CreateSet(inHosts) if err != nil { return err @@ -104,6 +104,10 @@ func (resp *sysResponse) getAbsentHostsRanksErrors() error { return nil } +func (resp *sysResponse) getErrors(errIn error) error { + return concatSysErrs(resp.getAbsentHostsRanksErrors(), errIn) +} + // SystemJoinReq contains the inputs for the system join request. type SystemJoinReq struct { unaryRequest @@ -233,14 +237,16 @@ func (req *SystemQueryReq) getStateMask() (system.MemberState, error) { // SystemQueryResp contains the request response. type SystemQueryResp struct { - sysResponse - Members system.Members `json:"members"` - Providers []string `json:"providers"` + sysResponse `json:"-"` + Members system.Members `json:"members"` + Providers []string `json:"providers"` } -// UnmarshalJSON unpacks JSON message into SystemQueryResp struct. -func (resp *SystemQueryResp) UnmarshalJSON(data []byte) error { - type Alias SystemQueryResp +// Wrap sysResponse handling of absent hosts and ranks in a helper to be called from response +// UnmarshalJSON implementations. +func unmarshalSysRespJsonFields(data []byte, sr *sysResponse) error { + resp := &sysResponse{} + type Alias sysResponse aux := &struct { AbsentHosts string AbsentRanks string @@ -251,7 +257,22 @@ func (resp *SystemQueryResp) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &aux); err != nil { return err } - if err := resp.getAbsentHostsRanks(aux.AbsentHosts, aux.AbsentRanks); err != nil { + if err := sr.setAbsentHostsRanks(aux.AbsentHosts, aux.AbsentRanks); err != nil { + return err + } + return nil +} + +// UnmarshalJSON unpacks JSON message into SystemQueryResp struct. +func (resp *SystemQueryResp) UnmarshalJSON(data []byte) error { + type Alias SystemQueryResp + aux := &struct{ *Alias }{Alias: (*Alias)(resp)} + + // Use type alias to avoid recursive decode issues. + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + if err := unmarshalSysRespJsonFields(data, &resp.sysResponse); err != nil { return err } @@ -261,7 +282,7 @@ func (resp *SystemQueryResp) UnmarshalJSON(data []byte) error { // Errors returns a single error combining all error messages associated with a // system query response. func (resp *SystemQueryResp) Errors() error { - return resp.getAbsentHostsRanksErrors() + return resp.sysResponse.getErrors(nil) } // SystemQuery requests DAOS system status. @@ -340,24 +361,20 @@ type SystemStartReq struct { // SystemStartResp contains the request response. type SystemStartResp struct { - sysResponse - Results system.MemberResults // resulting from harness starts + sysResponse `json:"-"` + Results system.MemberResults // resulting from harness starts } // UnmarshalJSON unpacks JSON message into SystemStartResp struct. func (resp *SystemStartResp) UnmarshalJSON(data []byte) error { type Alias SystemStartResp - aux := &struct { - AbsentHosts string - AbsentRanks string - *Alias - }{ - Alias: (*Alias)(resp), - } + aux := &struct{ *Alias }{Alias: (*Alias)(resp)} + + // Use type alias to avoid recursive decode issues. if err := json.Unmarshal(data, &aux); err != nil { return err } - if err := resp.getAbsentHostsRanks(aux.AbsentHosts, aux.AbsentRanks); err != nil { + if err := unmarshalSysRespJsonFields(data, &resp.sysResponse); err != nil { return err } @@ -367,7 +384,7 @@ func (resp *SystemStartResp) UnmarshalJSON(data []byte) error { // Errors returns a single error combining all error messages associated with a // system start response. func (resp *SystemStartResp) Errors() error { - return concatSysErrs(resp.getAbsentHostsRanksErrors(), resp.Results.Errors()) + return resp.sysResponse.getErrors(resp.Results.Errors()) } // SystemStart will perform a start after a controlled shutdown of DAOS system. @@ -410,24 +427,20 @@ type SystemStopReq struct { // SystemStopResp contains the request response. type SystemStopResp struct { - sysResponse - Results system.MemberResults + sysResponse `json:"-"` + Results system.MemberResults } // UnmarshalJSON unpacks JSON message into SystemStopResp struct. func (resp *SystemStopResp) UnmarshalJSON(data []byte) error { type Alias SystemStopResp - aux := &struct { - AbsentHosts string - AbsentRanks string - *Alias - }{ - Alias: (*Alias)(resp), - } + aux := &struct{ *Alias }{Alias: (*Alias)(resp)} + + // Use type alias to avoid recursive decode issues. if err := json.Unmarshal(data, &aux); err != nil { return err } - if err := resp.getAbsentHostsRanks(aux.AbsentHosts, aux.AbsentRanks); err != nil { + if err := unmarshalSysRespJsonFields(data, &resp.sysResponse); err != nil { return err } @@ -437,7 +450,7 @@ func (resp *SystemStopResp) UnmarshalJSON(data []byte) error { // Errors returns a single error combining all error messages associated with a // system stop response. func (resp *SystemStopResp) Errors() error { - return concatSysErrs(resp.getAbsentHostsRanksErrors(), resp.Results.Errors()) + return resp.sysResponse.getErrors(resp.Results.Errors()) } // SystemStop will perform a two-phase controlled shutdown of DAOS system and a @@ -452,12 +465,12 @@ func SystemStop(ctx context.Context, rpcClient UnaryInvoker, req *SystemStopReq) return nil, errors.Errorf("nil %T request", req) } - pbReq := new(mgmtpb.SystemStopReq) - pbReq.Hosts = req.Hosts.String() - pbReq.Ranks = req.Ranks.String() - pbReq.Force = req.Force - pbReq.Sys = req.getSystem(rpcClient) - + pbReq := &mgmtpb.SystemStopReq{ + Hosts: req.Hosts.String(), + Ranks: req.Ranks.String(), + Sys: req.getSystem(rpcClient), + Force: req.Force, + } req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return mgmtpb.NewMgmtSvcClient(conn).SystemStop(ctx, pbReq) }) @@ -508,16 +521,18 @@ type SystemExcludeReq struct { Clear bool } -// SystemExcludeResp contains the request response. +// SystemExcludeResp contains the request response. UnmarshalJSON is not implemented on this type +// because missing ranks or hosts specified in requests are not tolerated and therefore not returned +// in the response so decoding is not required. type SystemExcludeResp struct { - sysResponse - Results system.MemberResults + sysResponse `json:"-"` + Results system.MemberResults } -// Errors returns a single error combining all error messages associated with a -// system exclude response. +// Errors returns a single error combining all error messages associated with a system exclude +// response. func (resp *SystemExcludeResp) Errors() error { - return concatSysErrs(resp.getAbsentHostsRanksErrors(), resp.Results.Errors()) + return resp.sysResponse.getErrors(resp.Results.Errors()) } // SystemExclude will mark the specified ranks as administratively excluded from the system. @@ -546,6 +561,75 @@ func SystemExclude(ctx context.Context, rpcClient UnaryInvoker, req *SystemExclu return resp, convertMSResponse(ur, resp) } +// SystemDrainReq contains the inputs for the system drain request. +type SystemDrainReq struct { + unaryRequest + msRequest + sysRequest +} + +// DrainResult describes the result of a drain operation on a pool's ranks. +type DrainResult struct { + Status int32 `json:"status"` // Status returned from a specific drain call + Msg string `json:"msg"` // Error message if Status is not Success + PoolID string `json:"pool_id"` // Unique identifier for pool + Ranks string `json:"ranks"` // RankSet of ranks that should be drained on pool +} + +// SystemDrainResp contains the request response. UnmarshalJSON is not implemented on this type +// because missing ranks or hosts specified in requests are not tolerated and therefore not returned +// in the response so decoding is not required. +type SystemDrainResp struct { + sysResponse `json:"-"` + Results []*DrainResult `json:"results"` +} + +// Errors returns a single error combining all error messages associated with a system drain +// response. +func (resp *SystemDrainResp) Errors() error { + out := new(strings.Builder) + + for _, r := range resp.Results { + if r.Status != int32(daos.Success) { + fmt.Fprintf(out, "%s\n", r.Msg) + } + } + + var err error + if out.String() != "" { + err = errors.New(out.String()) + } + + return resp.sysResponse.getErrors(err) +} + +// SystemDrain will drain either hosts or ranks from all pools that they are members of. When hosts +// are specified in the request, any ranks that are resident on that host are drained from all +// relevant pools. +func SystemDrain(ctx context.Context, rpcClient UnaryInvoker, req *SystemDrainReq) (*SystemDrainResp, error) { + if req == nil { + return nil, errors.Errorf("nil %T request", req) + } + + pbReq := &mgmtpb.SystemDrainReq{ + Hosts: req.Hosts.String(), + Ranks: req.Ranks.String(), + Sys: req.getSystem(rpcClient), + } + req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { + return mgmtpb.NewMgmtSvcClient(conn).SystemDrain(ctx, pbReq) + }) + + rpcClient.Debugf("DAOS system drain request: %s", pbUtil.Debug(pbReq)) + ur, err := rpcClient.InvokeUnaryRPC(ctx, req) + if err != nil { + return nil, err + } + + resp := new(SystemDrainResp) + return resp, convertMSResponse(ur, resp) +} + // SystemEraseReq contains the inputs for a system erase request. type SystemEraseReq struct { msRequest diff --git a/src/control/security/grpc_authorization.go b/src/control/security/grpc_authorization.go index 2fc913a20e7..dc80a114921 100644 --- a/src/control/security/grpc_authorization.go +++ b/src/control/security/grpc_authorization.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -51,6 +51,7 @@ var methodAuthorizations = map[string][]Component{ "/mgmt.MgmtSvc/SystemStart": {ComponentAdmin}, "/mgmt.MgmtSvc/SystemStop": {ComponentAdmin}, "/mgmt.MgmtSvc/SystemExclude": {ComponentAdmin}, + "/mgmt.MgmtSvc/SystemDrain": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolCreate": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolDestroy": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolQuery": {ComponentAdmin}, diff --git a/src/control/security/grpc_authorization_test.go b/src/control/security/grpc_authorization_test.go index d054c54ed08..cbb6026ea0f 100644 --- a/src/control/security/grpc_authorization_test.go +++ b/src/control/security/grpc_authorization_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -76,6 +76,7 @@ func TestSecurity_ComponentHasAccess(t *testing.T) { "/mgmt.MgmtSvc/SystemErase": {ComponentAdmin}, "/mgmt.MgmtSvc/SystemStart": {ComponentAdmin}, "/mgmt.MgmtSvc/SystemExclude": {ComponentAdmin}, + "/mgmt.MgmtSvc/SystemDrain": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolCreate": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolDestroy": {ComponentAdmin}, "/mgmt.MgmtSvc/PoolQuery": {ComponentAdmin}, diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index 536b3397a42..c4ce6aed44c 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -1072,6 +1072,105 @@ func (svc *mgmtSvc) SystemExclude(ctx context.Context, req *mgmtpb.SystemExclude return resp, nil } +func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) (*mgmtpb.SystemDrainResp, error) { + if err := svc.checkLeaderRequest(wrapCheckerReq(req)); err != nil { + return nil, err + } + + if req.Hosts == "" && req.Ranks == "" { + return nil, errors.New("no hosts or ranks specified") + } + + hitRanks, missRanks, missHosts, err := svc.resolveRanks(req.Hosts, req.Ranks) + if err != nil { + return nil, err + } + + if missHosts.Count() > 0 { + return nil, errors.Errorf("invalid host(s): %s", missHosts.String()) + } + if missRanks.Count() > 0 { + return nil, errors.Errorf("invalid rank(s): %s", missRanks.String()) + } + if hitRanks.Count() == 0 { + return nil, errors.New("no ranks to drain") + } + + // Drain rank on each pool it belongs to. + + psList, err := svc.sysdb.PoolServiceList(false) + if err != nil { + return nil, err + } + + resp := new(mgmtpb.SystemDrainResp) + drainReq := new(mgmtpb.PoolDrainReq) + drainReq.Sys = req.Sys + + ranksMap := make(map[ranklist.Rank]struct{}) + for _, r := range hitRanks.Ranks() { + ranksMap[r] = struct{}{} + } + + poolRanks := make(map[string]*ranklist.RankSet) + for _, ps := range psList { + for _, r := range ps.Storage.CurrentRanks() { + if _, exists := ranksMap[r]; !exists { + continue + } + id := ps.PoolUUID.String() + if _, exists := poolRanks[id]; !exists { + poolRanks[id] = ranklist.MustCreateRankSet("") + } + poolRanks[id].Add(r) + } + } + + for id, rs := range poolRanks { + if rs.Count() == 0 { + continue + } + + // Use our incoming request and just replace relevant parameters on each iteration. + drainReq.Id = id + + // TODO DAOS-6611: Drain multiple pool-ranks per call when drpc.MethodPoolDrain API + // supports it. + for _, r := range rs.Ranks() { + var errMsg string + drainReq.Rank = r.Uint32() + + drpcResp, err := svc.makeLockedPoolServiceCall(ctx, drpc.MethodPoolDrain, + drainReq) + if err != nil { + return nil, err + } + + drainResp := &mgmtpb.PoolDrainResp{} + if err = proto.Unmarshal(drpcResp.Body, drainResp); err != nil { + errMsg = errors.Wrap(err, "unmarshal PoolEvict response").Error() + drainResp.Status = int32(daos.IOInvalid) + } else if drainResp.Status != int32(daos.Success) { + errMsg = fmt.Sprintf("drain failed for rank %d on pool %s: %s", + drainReq.Rank, drainReq.Id, + daos.Status(drainResp.Status).Error()) + } + + svc.log.Tracef("pool-drain triggered from system-drain: '%+v' (req: '%+v')", + drainResp, drainReq) + + resp.Results = append(resp.Results, &mgmtpb.SystemDrainResp_DrainResult{ + Status: drainResp.Status, + Msg: errMsg, + PoolId: drainReq.Id, + Ranks: fmt.Sprintf("%d", drainReq.Rank), + }) + } + } + + return resp, nil +} + // ClusterEvent management service gRPC handler receives ClusterEvent requests // from control-plane instances attempting to notify the MS of a cluster event // in the DAOS system (this handler should only get called on the MS leader). @@ -1221,7 +1320,7 @@ func (svc *mgmtSvc) SystemCleanup(ctx context.Context, req *mgmtpb.SystemCleanup evictReq.Machine = req.Machine for _, ps := range psList { - var errmsg string = "" + var errMsg string // Use our incoming request and just replace the uuid on each iteration evictReq.Id = ps.PoolUUID.String() @@ -1234,18 +1333,18 @@ func (svc *mgmtSvc) SystemCleanup(ctx context.Context, req *mgmtpb.SystemCleanup res := &mgmtpb.PoolEvictResp{} if err = proto.Unmarshal(dresp.Body, res); err != nil { res.Status = int32(daos.IOInvalid) - errmsg = errors.Wrap(err, "unmarshal PoolEvict response").Error() + errMsg = errors.Wrap(err, "unmarshal PoolEvict response").Error() res.Count = 0 } if res.Status != int32(daos.Success) { - errmsg = fmt.Sprintf("Unable to clean up handles for machine %s on pool %s", evictReq.Machine, evictReq.Id) + errMsg = fmt.Sprintf("Unable to clean up handles for machine %s on pool %s", evictReq.Machine, evictReq.Id) } svc.log.Debugf("Response from pool evict in cleanup: '%+v' (req: '%+v')", res, evictReq) resp.Results = append(resp.Results, &mgmtpb.SystemCleanupResp_CleanupResult{ Status: res.Status, - Msg: errmsg, + Msg: errMsg, PoolId: evictReq.Id, Count: uint32(res.Count), }) diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 64c28c455db..a2f7005db45 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -849,6 +849,7 @@ ds_mgmt_drpc_pool_drain(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) mgmt__pool_drain_req__free_unpacked(req, &alloc.alloc); } + void ds_mgmt_drpc_pool_extend(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) { diff --git a/src/proto/mgmt/mgmt.proto b/src/proto/mgmt/mgmt.proto index 82a83aa36f9..c0b8c13919d 100644 --- a/src/proto/mgmt/mgmt.proto +++ b/src/proto/mgmt/mgmt.proto @@ -79,6 +79,8 @@ service MgmtSvc { rpc SystemStart(SystemStartReq) returns(SystemStartResp) {} // Exclude DAOS ranks rpc SystemExclude(SystemExcludeReq) returns(SystemExcludeResp) {} + // Drain DAOS ranks from all pools + rpc SystemDrain(SystemDrainReq) returns (SystemDrainResp) {} // Erase DAOS system database prior to reformat rpc SystemErase(SystemEraseReq) returns(SystemEraseResp) {} // Clean up leaked resources for a given node diff --git a/src/proto/mgmt/system.proto b/src/proto/mgmt/system.proto index 80ba956afeb..4ee1e7cbf65 100644 --- a/src/proto/mgmt/system.proto +++ b/src/proto/mgmt/system.proto @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -78,6 +78,27 @@ message SystemExcludeResp { repeated shared.RankResult results = 1; } +// SystemDrainReq supplies system-drain parameters. +message SystemDrainReq +{ + string sys = 1; // DAOS system name + string ranks = 2; // rankset to drain on all pools + string hosts = 3; // hostset to drain on all pools +} + +// SystemDrainResp returns status of system-drain request. +message SystemDrainResp +{ + message DrainResult + { + int32 status = 1; // Status of the evict on the specific pool + string msg = 2; // Error message if status indicates an error + string pool_id = 3; // uuid of pool + string ranks = 4; // rankset to have been drained on pool + } + repeated DrainResult results = 1; // Results for individual pool-ranks drain calls. +} + // SystemQueryReq supplies system query parameters. message SystemQueryReq { string sys = 1; // DAOS system name @@ -116,8 +137,7 @@ message SystemCleanupResp { int32 status = 1; // Status of the evict on the specific pool string msg = 2; // Error message if status indicates an error string pool_id = 3; // uuid of pool - uint32 count = 4; // number of pool handles cleaned up - + uint32 count = 4; // number of pool handles cleaned up } repeated CleanupResult results = 1; // Results and Status for individual pools that are cleanedup. } From 18ec09e0799a2bc569fbf2e18a3a0e5f3bb3599b Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Thu, 21 Nov 2024 13:24:59 +0000 Subject: [PATCH 2/6] improve and expand unit tests in lib/system and cmd/dmg dirs Features: control Required-githooks: true Signed-off-by: Tom Nabarro --- docs/admin/pool_operations.md | 17 + src/control/cmd/dmg/system.go | 92 ++- src/control/cmd/dmg/system_test.go | 358 ++------ src/control/lib/control/system.go | 75 +- src/control/lib/control/system_test.go | 1042 +++++++++++++++--------- src/control/server/mgmt_system.go | 2 +- 6 files changed, 818 insertions(+), 768 deletions(-) diff --git a/docs/admin/pool_operations.md b/docs/admin/pool_operations.md index d423f6a9b65..65f57c0fb70 100644 --- a/docs/admin/pool_operations.md +++ b/docs/admin/pool_operations.md @@ -1260,6 +1260,23 @@ The pool target drain command accepts 2 parameters: * The engine rank of the target(s) to be drained. * The target indices of the targets to be drained from that engine rank (optional). +#### System Drain + +To drain ranks or hosts from all pools that they belong to, the 'dmg system drain' +command can be used. The command takes either a host-set or rank-set: + +To drain a set of hosts from all pools (drains all ranks on selected hosts): + +```Bash +$ dmg system drain --hosts foo-[001-100] +``` + +To drain a set of ranks from all pools: + +```Bash +$ dmg system drain --ranks 1-100 +``` + ### Reintegration After an engine failure and exclusion, an operator can fix the underlying issue diff --git a/src/control/cmd/dmg/system.go b/src/control/cmd/dmg/system.go index 8dafb947f58..ee5213934d9 100644 --- a/src/control/cmd/dmg/system.go +++ b/src/control/cmd/dmg/system.go @@ -23,6 +23,8 @@ import ( "github.com/daos-stack/daos/src/control/lib/ui" ) +var errNoRanks = errors.New("no ranks or hosts specified") + // SystemCmd is the struct representing the top-level system subcommand. type SystemCmd struct { LeaderQuery leaderQueryCmd `command:"leader-query" description:"Query for current Management Service leader"` @@ -34,7 +36,7 @@ type SystemCmd struct { Drain systemDrainCmd `command:"drain" description:"Drain ranks or hosts from all relevant pools in DAOS system"` Erase systemEraseCmd `command:"erase" description:"Erase system metadata prior to reformat"` ListPools PoolListCmd `command:"list-pools" description:"List all pools in the DAOS system"` - Cleanu systemCleanupCmd `command:"cleanup" description:"Clean up all resources associated with the specified machine"` + Cleanup systemCleanupCmd `command:"cleanup" description:"Clean up all resources associated with the specified machine"` SetAttr systemSetAttrCmd `command:"set-attr" description:"Set system attributes"` GetAttr systemGetAttrCmd `command:"get-attr" description:"Get system attributes"` DelAttr systemDelAttrCmd `command:"del-attr" description:"Delete system attributes"` @@ -212,6 +214,46 @@ func (cmd *systemStopCmd) Execute(_ []string) (errOut error) { return resp.Errors() } +// systemStartCmd is the struct representing the command to start system. +type systemStartCmd struct { + baseRankListCmd +} + +// Execute is run when systemStartCmd activates. +func (cmd *systemStartCmd) Execute(_ []string) (errOut error) { + defer func() { + errOut = errors.Wrap(errOut, "system start failed") + }() + + if err := cmd.validateHostsRanks(); err != nil { + return err + } + + req := new(control.SystemStartReq) + req.Hosts.Replace(&cmd.Hosts.HostSet) + req.Ranks.Replace(&cmd.Ranks.RankSet) + + resp, err := control.SystemStart(cmd.MustLogCtx(), cmd.ctlInvoker, req) + if err != nil { + return err // control api returned an error, disregard response + } + + if cmd.JSONOutputEnabled() { + return cmd.OutputJSON(resp, resp.Errors()) + } + + var out, outErr strings.Builder + if err := pretty.PrintSystemStartResponse(&out, &outErr, resp); err != nil { + return err + } + cmd.Info(out.String()) + if outErr.String() != "" { + cmd.Error(outErr.String()) + } + + return resp.Errors() +} + type systemExcludeCmd struct { baseRankListCmd } @@ -221,7 +263,7 @@ func (cmd *systemExcludeCmd) execute(clear bool) error { return err } if cmd.Ranks.Count() == 0 && cmd.Hosts.Count() == 0 { - return errors.New("no ranks or hosts specified") + return errNoRanks } req := &control.SystemExcludeReq{Clear: clear} @@ -276,7 +318,7 @@ func (cmd *systemDrainCmd) Execute(_ []string) (errOut error) { return err } if cmd.Ranks.Count() == 0 && cmd.Hosts.Count() == 0 { - return errors.New("no ranks or hosts specified") + return errNoRanks } req := new(control.SystemDrainReq) @@ -308,49 +350,10 @@ func (cmd *systemDrainCmd) Execute(_ []string) (errOut error) { return nil } -// systemStartCmd is the struct representing the command to start system. -type systemStartCmd struct { - baseRankListCmd -} - -// Execute is run when systemStartCmd activates. -func (cmd *systemStartCmd) Execute(_ []string) (errOut error) { - defer func() { - errOut = errors.Wrap(errOut, "system start failed") - }() - - if err := cmd.validateHostsRanks(); err != nil { - return err - } - req := new(control.SystemStartReq) - req.Hosts.Replace(&cmd.Hosts.HostSet) - req.Ranks.Replace(&cmd.Ranks.RankSet) - - resp, err := control.SystemStart(cmd.MustLogCtx(), cmd.ctlInvoker, req) - if err != nil { - return err // control api returned an error, disregard response - } - - if cmd.JSONOutputEnabled() { - return cmd.OutputJSON(resp, resp.Errors()) - } - - var out, outErr strings.Builder - if err := pretty.PrintSystemStartResponse(&out, &outErr, resp); err != nil { - return err - } - cmd.Info(out.String()) - if outErr.String() != "" { - cmd.Error(outErr.String()) - } - - return resp.Errors() -} - type systemCleanupCmd struct { baseCtlCmd Args struct { - Machine string `positional-arg-name:""` + Machine string `positional-arg-name:"machine to cleanup" required:"1"` } `positional-args:"yes"` Verbose bool `long:"verbose" short:"v" description:"Output additional cleanup information"` } @@ -360,12 +363,11 @@ func (cmd *systemCleanupCmd) Execute(_ []string) (errOut error) { errOut = errors.Wrap(errOut, "system cleanup failed") }() - ctx := cmd.MustLogCtx() req := new(control.SystemCleanupReq) req.SetSystem(cmd.config.SystemName) req.Machine = cmd.Args.Machine - resp, err := control.SystemCleanup(ctx, cmd.ctlInvoker, req) + resp, err := control.SystemCleanup(cmd.MustLogCtx(), cmd.ctlInvoker, req) if err != nil { return err // control api returned an error, disregard response } diff --git a/src/control/cmd/dmg/system_test.go b/src/control/cmd/dmg/system_test.go index be8b3ab6dd2..a1cc47c2c0e 100644 --- a/src/control/cmd/dmg/system_test.go +++ b/src/control/cmd/dmg/system_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -41,6 +41,14 @@ func TestDmg_SystemCommands(t *testing.T) { return req } + withSystem := func(req control.UnaryRequest, sys string) control.UnaryRequest { + if rs, ok := req.(interface{ SetSystem(string) }); ok { + rs.SetSystem(sys) + } + + return req + } + runCmdTests(t, []cmdTest{ { "system query with no arguments", @@ -236,6 +244,52 @@ func TestDmg_SystemCommands(t *testing.T) { "", errors.New("--ranks and --rank-hosts options cannot be set together"), }, + { + "system exclude with multiple ranks", + "system exclude --ranks 0,1,4", + strings.Join([]string{ + printRequest(t, withRanks(&control.SystemExcludeReq{}, 0, 1, 4)), + }, " "), + nil, + }, + { + "system exclude with no ranks", + "system exclude", + "", + errNoRanks, + }, + { + "system drain with multiple ranks", + "system drain --ranks 0,1,4 -v", + strings.Join([]string{ + printRequest(t, withSystem( + withRanks(&control.SystemDrainReq{}, 0, 1, 4), + "daos_server")), + }, " "), + nil, + }, + { + "system drain without ranks", + "system drain -v", + "", + errNoRanks, + }, + { + "system cleanup with machine name", + "system cleanup foo1 -v", + strings.Join([]string{ + printRequest(t, withSystem(&control.SystemCleanupReq{ + Machine: "foo1", + }, "daos_server")), + }, " "), + nil, + }, + { + "system cleanup without machine name", + "system cleanup -v", + "", + errors.New("not provided"), + }, { "leader query", "system leader-query", @@ -317,26 +371,26 @@ func TestDmg_SystemCommands(t *testing.T) { }) } -func TestDmg_LeaderQueryCmd_Errors(t *testing.T) { +func TestDmg_leaderQueryCmd(t *testing.T) { for name, tc := range map[string]struct { ctlCfg *control.Config resp *mgmtpb.LeaderQueryResp msErr error expErr error }{ - "list pools no config": { + "no config": { resp: &mgmtpb.LeaderQueryResp{}, expErr: errors.New("leader query failed: no configuration loaded"), }, - "list pools success": { + "success": { ctlCfg: &control.Config{}, resp: &mgmtpb.LeaderQueryResp{}, }, - "list pools ms failures": { + "ms failures": { ctlCfg: &control.Config{}, resp: &mgmtpb.LeaderQueryResp{}, - msErr: errors.New("remote failed"), - expErr: errors.New("remote failed"), + msErr: errors.New("failed"), + expErr: errors.New("failed"), }, } { t.Run(name, func(t *testing.T) { @@ -359,135 +413,16 @@ func TestDmg_LeaderQueryCmd_Errors(t *testing.T) { } } -func TestDmg_systemQueryCmd_Errors(t *testing.T) { - for name, tc := range map[string]struct { - resp *mgmtpb.SystemQueryResp - msErr error - expErr error - }{ - "system query success": { - resp: &mgmtpb.SystemQueryResp{ - Members: []*mgmtpb.SystemMember{ - { - Rank: 1, - Uuid: test.MockUUID(1), - State: system.MemberStateReady.String(), - Addr: "10.0.0.1:10001", - }, - { - Rank: 2, - Uuid: test.MockUUID(2), - State: system.MemberStateReady.String(), - Addr: "10.0.0.1:10001", - }, - }, - }, - }, - "system query ms failures": { - msErr: errors.New("remote failed"), - expErr: errors.New("remote failed"), - }, - "system query absent hosts": { - resp: &mgmtpb.SystemQueryResp{ - Absenthosts: "foo[1-23]", - }, - expErr: errors.New("system query failed: non-existent hosts"), - }, - "system query absent ranks": { - resp: &mgmtpb.SystemQueryResp{ - Absentranks: "1-23", - }, - expErr: errors.New("system query failed: non-existent ranks"), - }, - "system query absent hosts and ranks": { - resp: &mgmtpb.SystemQueryResp{ - Absenthosts: "foo[1-23]", - Absentranks: "1-23", - }, - expErr: errors.New("system query failed: non-existent hosts foo[1-23], " + - "non-existent ranks 1-23"), - }, - } { - t.Run(name, func(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - mi := control.NewMockInvoker(log, &control.MockInvokerConfig{ - UnaryResponse: control.MockMSResponse("10.0.0.1:10001", - tc.msErr, tc.resp), - }) - - queryCmd := new(systemQueryCmd) - queryCmd.setInvoker(mi) - queryCmd.SetLog(log) - - gotErr := queryCmd.Execute(nil) - test.CmpErr(t, tc.expErr, gotErr) - }) - } -} - -func TestDmg_systemStartCmd_Errors(t *testing.T) { +// TestDmg_systemStartCmd covers case where duplicate rank result is detected when constructing +// rank groups, this applies across start stop exclude and any other commands that receive rank +// results in response but the test case isn't run individually for all those commands that share +// the behavior. +func TestDmg_systemStartCmd(t *testing.T) { for name, tc := range map[string]struct { resp *mgmtpb.SystemStartResp msErr error expErr error }{ - "system start success": { - resp: &mgmtpb.SystemStartResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 2, Action: "start", - State: system.MemberStateReady.String(), - }, - { - Rank: 3, Action: "start", - State: system.MemberStateReady.String(), - }, - }, - }, - }, - "system start ms failures": { - resp: &mgmtpb.SystemStartResp{ - Absenthosts: "foo[1-23]", - Absentranks: "1-23", - Results: []*sharedpb.RankResult{ - { - Rank: 24, Action: "start", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - msErr: errors.New("remote failed"), - expErr: errors.New("remote failed"), - }, - "system start rank failures": { - resp: &mgmtpb.SystemStartResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 2, Action: "start", - State: system.MemberStateReady.String(), - }, - { - Rank: 3, Action: "start", - Errored: true, Msg: "uh oh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 5, Action: "start", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 4, Action: "start", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - expErr: errors.New("system start failed: check results for failed ranks 3-5"), - }, "system start duplicate rank results": { resp: &mgmtpb.SystemStartResp{ Results: []*sharedpb.RankResult{ @@ -504,38 +439,6 @@ func TestDmg_systemStartCmd_Errors(t *testing.T) { }, expErr: errors.New("duplicate result for rank"), }, - "system start absent hosts": { - resp: &mgmtpb.SystemStartResp{ - Absenthosts: "foo[1-23]", - }, - expErr: errors.New("system start failed: non-existent hosts"), - }, - "system start absent ranks": { - resp: &mgmtpb.SystemStartResp{ - Absentranks: "1-23", - }, - expErr: errors.New("system start failed: non-existent ranks"), - }, - "system start absent hosts and ranks and failed ranks": { - resp: &mgmtpb.SystemStartResp{ - Absenthosts: "foo[1-23]", - Absentranks: "1-23", - Results: []*sharedpb.RankResult{ - { - Rank: 24, Action: "start", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 25, Action: "start", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - expErr: errors.New("system start failed: non-existent hosts foo[1-23], " + - "non-existent ranks 1-23, check results for failed ranks 24-2"), - }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) @@ -555,132 +458,3 @@ func TestDmg_systemStartCmd_Errors(t *testing.T) { }) } } - -func TestDmg_systemStopCmd_Errors(t *testing.T) { - for name, tc := range map[string]struct { - resp *mgmtpb.SystemStopResp - msErr error - expErr error - }{ - "system stop success": { - resp: &mgmtpb.SystemStopResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 2, Action: "stop", - State: system.MemberStateStopped.String(), - }, - { - Rank: 3, Action: "stop", - State: system.MemberStateStopped.String(), - }, - }, - }, - }, - "system stop ms failures": { - resp: &mgmtpb.SystemStopResp{ - Absenthosts: "foo[1-23]", - Absentranks: "1-23", - Results: []*sharedpb.RankResult{ - { - Rank: 24, Action: "stop", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - msErr: errors.New("remote failed"), - expErr: errors.New("remote failed"), - }, - "system stop rank failures": { - resp: &mgmtpb.SystemStopResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 2, Action: "stop", - State: system.MemberStateStopped.String(), - }, - { - Rank: 3, Action: "stop", - Errored: true, Msg: "uh oh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 5, Action: "stop", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 4, Action: "stop", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - expErr: errors.New("system stop failed: check results for failed ranks 3-5"), - }, - "system stop duplicate rank results": { - resp: &mgmtpb.SystemStopResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 2, Action: "stop", - State: system.MemberStateStopped.String(), - }, - { - Rank: 2, Action: "stop", - Errored: true, Msg: "uh oh", - State: system.MemberStateErrored.String(), - }, - }, - }, - expErr: errors.New("duplicate result for rank"), - }, - "system stop absent hosts": { - resp: &mgmtpb.SystemStopResp{ - Absenthosts: "foo[1-23]", - }, - expErr: errors.New("system stop failed: non-existent hosts"), - }, - "system stop absent ranks": { - resp: &mgmtpb.SystemStopResp{ - Absentranks: "1-23", - }, - expErr: errors.New("system stop failed: non-existent ranks"), - }, - "system stop absent hosts and ranks and failed ranks": { - resp: &mgmtpb.SystemStopResp{ - Absenthosts: "foo[1-23]", - Absentranks: "1-23", - Results: []*sharedpb.RankResult{ - { - Rank: 24, Action: "stop", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - { - Rank: 25, Action: "stop", - Errored: true, Msg: "uh ohh", - State: system.MemberStateErrored.String(), - }, - }, - }, - expErr: errors.New("system stop failed: non-existent hosts foo[1-23], " + - "non-existent ranks 1-23, check results for failed ranks 24-2"), - }, - } { - t.Run(name, func(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - mi := control.NewMockInvoker(log, &control.MockInvokerConfig{ - UnaryResponse: control.MockMSResponse("10.0.0.1:10001", - tc.msErr, tc.resp), - }) - - stopCmd := new(systemStopCmd) - stopCmd.setInvoker(mi) - stopCmd.SetLog(log) - - gotErr := stopCmd.Execute(nil) - test.CmpErr(t, tc.expErr, gotErr) - }) - } -} diff --git a/src/control/lib/control/system.go b/src/control/lib/control/system.go index 876c2ed1eee..d87c2df3b25 100644 --- a/src/control/lib/control/system.go +++ b/src/control/lib/control/system.go @@ -9,7 +9,6 @@ package control import ( "context" "encoding/json" - "fmt" "net" "strings" "time" @@ -105,7 +104,11 @@ func (resp *sysResponse) getAbsentHostsRanksErrors() error { } func (resp *sysResponse) getErrors(errIn error) error { - return concatSysErrs(resp.getAbsentHostsRanksErrors(), errIn) + if errIn != nil { + errIn = errors.Errorf("check results for %s", errIn.Error()) + } + + return concatErrs(resp.getAbsentHostsRanksErrors(), errIn) } // SystemJoinReq contains the inputs for the system join request. @@ -296,10 +299,11 @@ func SystemQuery(ctx context.Context, rpcClient UnaryInvoker, req *SystemQueryRe return nil, errors.Errorf("nil %T request", req) } - pbReq := new(mgmtpb.SystemQueryReq) - pbReq.Hosts = req.Hosts.String() - pbReq.Ranks = req.Ranks.String() - pbReq.Sys = req.getSystem(rpcClient) + pbReq := &mgmtpb.SystemQueryReq{ + Hosts: req.Hosts.String(), + Ranks: req.Ranks.String(), + Sys: req.getSystem(rpcClient), + } mask, err := req.getStateMask() if err != nil { @@ -335,14 +339,14 @@ func SystemQuery(ctx context.Context, rpcClient UnaryInvoker, req *SystemQueryRe return resp, convertMSResponse(ur, resp) } -func concatSysErrs(errSys, errRes error) error { +func concatErrs(errOne, errTwo error) error { var errMsgs []string - if errSys != nil { - errMsgs = append(errMsgs, errSys.Error()) + if errOne != nil { + errMsgs = append(errMsgs, errOne.Error()) } - if errRes != nil { - errMsgs = append(errMsgs, "check results for "+errRes.Error()) + if errTwo != nil { + errMsgs = append(errMsgs, errTwo.Error()) } if len(errMsgs) > 0 { @@ -530,9 +534,10 @@ type SystemExcludeResp struct { } // Errors returns a single error combining all error messages associated with a system exclude -// response. +// response. Doesn't retrieve errors from sysResponse because missing ranks or hosts will not be +// populated in SystemExcludeResp. func (resp *SystemExcludeResp) Errors() error { - return resp.sysResponse.getErrors(resp.Results.Errors()) + return resp.Results.Errors() } // SystemExclude will mark the specified ranks as administratively excluded from the system. @@ -585,22 +590,16 @@ type SystemDrainResp struct { } // Errors returns a single error combining all error messages associated with a system drain -// response. -func (resp *SystemDrainResp) Errors() error { - out := new(strings.Builder) - - for _, r := range resp.Results { +// response. Doesn't retrieve errors from sysResponse because missing ranks or hosts will not be +// populated in SystemDrainResp. +func (sdr *SystemDrainResp) Errors() (errOut error) { + for _, r := range sdr.Results { if r.Status != int32(daos.Success) { - fmt.Fprintf(out, "%s\n", r.Msg) + errOut = concatErrs(errOut, errors.New(r.Msg)) } } - var err error - if out.String() != "" { - err = errors.New(out.String()) - } - - return resp.sysResponse.getErrors(err) + return } // SystemDrain will drain either hosts or ranks from all pools that they are members of. When hosts @@ -763,6 +762,10 @@ type LeaderQueryResp struct { // LeaderQuery requests the current Management Service leader and the set of // MS replicas. func LeaderQuery(ctx context.Context, rpcClient UnaryInvoker, req *LeaderQueryReq) (*LeaderQueryResp, error) { + if req == nil { + return nil, errors.Errorf("nil %T request", req) + } + pbReq := &mgmtpb.LeaderQueryReq{Sys: req.getSystem(rpcClient)} req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return mgmtpb.NewMgmtSvcClient(conn).LeaderQuery(ctx, pbReq) @@ -773,12 +776,15 @@ func LeaderQuery(ctx context.Context, rpcClient UnaryInvoker, req *LeaderQueryRe if err != nil { return nil, err } + rpcClient.Debugf("resp one: %+v", ur) resp := new(LeaderQueryResp) if err = convertMSResponse(ur, resp); err != nil { return nil, errors.Wrap(err, "converting MS to LeaderQuery resp") } + rpcClient.Debugf("resp one: %+v", resp) + req.SetHostList(resp.Replicas) ur, err = rpcClient.InvokeUnaryRPC(ctx, req) if err != nil { @@ -978,20 +984,14 @@ type SystemCleanupResp struct { // Errors returns a single error combining all error messages associated with a // system cleanup response. -func (scr *SystemCleanupResp) Errors() error { - out := new(strings.Builder) - +func (scr *SystemCleanupResp) Errors() (errOut error) { for _, r := range scr.Results { if r.Status != int32(daos.Success) { - fmt.Fprintf(out, "%s\n", r.Msg) + errOut = concatErrs(errOut, errors.New(r.Msg)) } } - if out.String() != "" { - return errors.New(out.String()) - } - - return nil + return } // SystemCleanup requests resources associated with a machine name be cleanedup. @@ -1004,9 +1004,10 @@ func SystemCleanup(ctx context.Context, rpcClient UnaryInvoker, req *SystemClean return nil, errors.New("SystemCleanup requires a machine name.") } - pbReq := new(mgmtpb.SystemCleanupReq) - pbReq.Machine = req.Machine - pbReq.Sys = req.getSystem(rpcClient) + pbReq := &mgmtpb.SystemCleanupReq{ + Sys: req.getSystem(rpcClient), + Machine: req.Machine, + } req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return mgmtpb.NewMgmtSvcClient(conn).SystemCleanup(ctx, pbReq) diff --git a/src/control/lib/control/system_test.go b/src/control/lib/control/system_test.go index 3a85eaa1d1a..6f1d43fe429 100644 --- a/src/control/lib/control/system_test.go +++ b/src/control/lib/control/system_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -490,6 +490,39 @@ func TestControl_SystemQueryReq_getStateMask(t *testing.T) { } } +func TestControl_sysResponse_getErrors(t *testing.T) { + for name, tc := range map[string]struct { + absentHosts string + absentRanks string + expErr error + }{ + "no errors": {}, + "absent hosts": { + absentHosts: "foo-[1-23]", + expErr: errors.New("non-existent hosts foo-[1-23]"), + }, + "absent ranks": { + absentRanks: "1-23", + expErr: errors.New("non-existent ranks 1-23"), + }, + "both absent hosts and ranks": { + absentHosts: "foo-[1-23]", + absentRanks: "1-23", + expErr: errors.New("non-existent hosts foo-[1-23], non-existent ranks 1-23"), + }, + } { + t.Run(name, func(t *testing.T) { + resp := new(sysResponse) + ahs := hostlist.MustCreateSet(tc.absentHosts) + resp.AbsentHosts.Replace(ahs) + ars := ranklist.MustCreateRankSet(tc.absentRanks) + resp.AbsentRanks.Replace(ars) + + test.CmpErr(t, tc.expErr, resp.getErrors(nil)) + }) + } +} + func TestControl_SystemQuery(t *testing.T) { testHS := hostlist.MustCreateSet("foo-[1-23]") testReqHS := new(SystemQueryReq) @@ -510,11 +543,12 @@ func TestControl_SystemQuery(t *testing.T) { } for name, tc := range map[string]struct { - req *SystemQueryReq - uErr error - uResp *UnaryResponse - expResp *SystemQueryResp - expErr error + req *SystemQueryReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemQueryResp + expRespErr error }{ "nil req": { req: nil, @@ -532,56 +566,54 @@ func TestControl_SystemQuery(t *testing.T) { }, "request absent host set": { req: testReqHS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemQueryResp{ - Absenthosts: "foo-[1-23]", - }), - expResp: testRespHS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemQueryResp{ + Absenthosts: "foo-[1-23]", + }), + expResp: testRespHS, + expRespErr: errors.New("non-existent hosts foo-[1-23]"), }, "request absent rank set": { req: testReqRS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemQueryResp{ - Absentranks: "1-23", - }), - expResp: testRespRS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemQueryResp{ + Absentranks: "1-23", + }), + expResp: testRespRS, + expRespErr: errors.New("non-existent ranks 1-23"), }, - "dual host dual rank": { + "multiple members": { req: new(SystemQueryReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemQueryResp{ - Members: []*mgmtpb.SystemMember{ - { - Rank: 1, - Uuid: test.MockUUID(1), - State: system.MemberStateReady.String(), - Addr: "10.0.0.1:10001", - FaultDomain: fdStrs[1], - }, - { - Rank: 2, - Uuid: test.MockUUID(2), - State: system.MemberStateReady.String(), - Addr: "10.0.0.1:10001", - FaultDomain: fdStrs[2], - }, - { - Rank: 0, - Uuid: test.MockUUID(0), - State: system.MemberStateStopped.String(), - Addr: "10.0.0.2:10001", - FaultDomain: fdStrs[0], - }, - { - Rank: 3, - Uuid: test.MockUUID(3), - State: system.MemberStateStopped.String(), - Addr: "10.0.0.2:10001", - FaultDomain: fdStrs[3], - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemQueryResp{ + Members: []*mgmtpb.SystemMember{ + { + Rank: 1, + Uuid: test.MockUUID(1), + State: system.MemberStateReady.String(), + Addr: "10.0.0.1:10001", + FaultDomain: fdStrs[1], + }, + { + Rank: 2, + Uuid: test.MockUUID(2), + State: system.MemberStateReady.String(), + Addr: "10.0.0.1:10001", + FaultDomain: fdStrs[2], + }, + { + Rank: 0, + Uuid: test.MockUUID(0), + State: system.MemberStateStopped.String(), + Addr: "10.0.0.2:10001", + FaultDomain: fdStrs[0], + }, + { + Rank: 3, + Uuid: test.MockUUID(3), + State: system.MemberStateStopped.String(), + Addr: "10.0.0.2:10001", + FaultDomain: fdStrs[3], }, }, - ), + }), expResp: &SystemQueryResp{ Members: system.Members{ system.MockMemberFullSpec(t, 1, test.MockUUID(1), "", @@ -599,6 +631,20 @@ func TestControl_SystemQuery(t *testing.T) { }, }, }, + "multiple members; with errors": { + req: new(SystemQueryReq), + uResp: MockMSResponse("host1", nil, &mgmtpb.SystemQueryResp{ + Absenthosts: "foo-[1-23]", + Absentranks: "1-23", + }), + expResp: func() *SystemQueryResp { + resp := new(SystemQueryResp) + resp.AbsentHosts.Replace(testHS) + resp.AbsentRanks.Replace(testRS) + return resp + }(), + expRespErr: errors.New("non-existent hosts foo-[1-23], non-existent ranks 1-23"), + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) @@ -622,45 +668,8 @@ func TestControl_SystemQuery(t *testing.T) { if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } - if diff := cmp.Diff(tc.expResp.AbsentHosts.String(), gotResp.AbsentHosts.String()); diff != "" { - t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) - } - if diff := cmp.Diff(tc.expResp.AbsentRanks.String(), gotResp.AbsentRanks.String()); diff != "" { - t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) - } - }) - } -} -func TestControl_SystemQueryRespErrors(t *testing.T) { - for name, tc := range map[string]struct { - absentHosts string - absentRanks string - expErr error - }{ - "no errors": {}, - "absent hosts": { - absentHosts: "foo-[1-23]", - expErr: errors.New("non-existent hosts foo-[1-23]"), - }, - "absent ranks": { - absentRanks: "1-23", - expErr: errors.New("non-existent ranks 1-23"), - }, - "both absent hosts and ranks": { - absentHosts: "foo-[1-23]", - absentRanks: "1-23", - expErr: errors.New("non-existent hosts foo-[1-23], non-existent ranks 1-23"), - }, - } { - t.Run(name, func(t *testing.T) { - resp := new(SystemQueryResp) - ahs := hostlist.MustCreateSet(tc.absentHosts) - resp.AbsentHosts.Replace(ahs) - ars := ranklist.MustCreateRankSet(tc.absentRanks) - resp.AbsentRanks.Replace(ars) - - test.CmpErr(t, tc.expErr, resp.Errors()) + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) }) } } @@ -679,11 +688,12 @@ func TestControl_SystemStart(t *testing.T) { testRespRS.AbsentRanks.Replace(testRS) for name, tc := range map[string]struct { - req *SystemStartReq - uErr error - uResp *UnaryResponse - expResp *SystemStartResp - expErr error + req *SystemStartReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemStartResp + expRespErr error }{ "nil req": { req: nil, @@ -701,52 +711,87 @@ func TestControl_SystemStart(t *testing.T) { }, "request absent host set": { req: testReqHS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemStartResp{ - Absenthosts: "foo-[1-23]", - }), - expResp: testRespHS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemStartResp{ + Absenthosts: "foo-[1-23]", + }), + expResp: testRespHS, + expRespErr: errors.New("non-existent hosts foo-[1-23]"), }, "request absent rank set": { req: testReqRS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemStartResp{ - Absentranks: "1-23", - }), - expResp: testRespRS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemStartResp{ + Absentranks: "1-23", + }), + expResp: testRespRS, + expRespErr: errors.New("non-existent ranks 1-23"), }, - "dual host dual rank": { + "multiple members": { req: new(SystemStartReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemStartResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 1, - State: system.MemberStateReady.String(), - }, - { - Rank: 2, - State: system.MemberStateReady.String(), - }, - { - Rank: 0, - State: system.MemberStateStopped.String(), - }, - { - Rank: 3, - State: system.MemberStateStopped.String(), - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemStartResp{ + Results: []*sharedpb.RankResult{ + { + Rank: 1, + State: system.MemberStateReady.String(), + }, + { + Rank: 2, + State: system.MemberStateReady.String(), + }, + { + Rank: 0, + State: system.MemberStateReady.String(), + }, + { + Rank: 3, + State: system.MemberStateReady.String(), }, }, - ), + }), expResp: &SystemStartResp{ Results: system.MemberResults{ system.NewMemberResult(1, nil, system.MemberStateReady), system.NewMemberResult(2, nil, system.MemberStateReady), - system.NewMemberResult(0, nil, system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), + system.NewMemberResult(0, nil, system.MemberStateReady), + system.NewMemberResult(3, nil, system.MemberStateReady), + }, + }, + }, + "multiple members; with errors": { + req: new(SystemStartReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemStartResp{ + Absenthosts: "foo-[1-23]", + Absentranks: "1-23", + Results: []*sharedpb.RankResult{ + { + Rank: 24, + State: system.MemberStateReady.String(), + }, + { + Rank: 25, Errored: true, Msg: "fail", + State: system.MemberStateErrored.String(), + }, + { + Rank: 0, Errored: true, Msg: "failed", + State: system.MemberStateErrored.String(), + }, + { + Rank: 26, + State: system.MemberStateReady.String(), + }, + }, + }), + expResp: &SystemStartResp{ + Results: system.MemberResults{ + system.NewMemberResult(24, nil, system.MemberStateReady), + system.NewMemberResult(25, errors.New("fail"), + system.MemberStateReady), + system.NewMemberResult(0, errors.New("failed"), + system.MemberStateReady), + system.NewMemberResult(26, nil, system.MemberStateReady), }, }, + expRespErr: errors.New("non-existent hosts foo-[1-23], " + + "non-existent ranks 1-23, check results for failed ranks 0,2"), }, } { t.Run(name, func(t *testing.T) { @@ -764,74 +809,14 @@ func TestControl_SystemStart(t *testing.T) { return } - cmpOpts := []cmp.Option{cmpopts.IgnoreUnexported(SystemStartResp{}, system.MemberResult{})} - if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { - t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) - } - if diff := cmp.Diff(tc.expResp.AbsentHosts.String(), gotResp.AbsentHosts.String()); diff != "" { - t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(SystemStartResp{}, system.MemberResult{}), } - if diff := cmp.Diff(tc.expResp.AbsentRanks.String(), gotResp.AbsentRanks.String()); diff != "" { + if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } - }) - } -} - -func TestControl_SystemStartRespErrors(t *testing.T) { - successResults := system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, nil, system.MemberStateReady), - system.NewMemberResult(0, nil, system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), - } - failedResults := system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, errors.New("fail"), system.MemberStateReady), - system.NewMemberResult(0, errors.New("failed"), system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), - } - - for name, tc := range map[string]struct { - absentHosts string - absentRanks string - results system.MemberResults - expErr error - }{ - "no errors": { - results: successResults, - }, - "absent hosts": { - absentHosts: "foo-[1-23]", - results: successResults, - expErr: errors.New("non-existent hosts foo-[1-23]"), - }, - "absent ranks": { - absentRanks: "1-23", - results: successResults, - expErr: errors.New("non-existent ranks 1-23"), - }, - "failed ranks": { - results: failedResults, - expErr: errors.New("check results for failed ranks 0,2"), - }, - "absent hosts and ranks with failed ranks": { - absentHosts: "foo-[1-23]", - absentRanks: "1-23", - results: failedResults, - expErr: errors.New("non-existent hosts foo-[1-23], " + - "non-existent ranks 1-23, check results for failed ranks 0,2"), - }, - } { - t.Run(name, func(t *testing.T) { - resp := new(SystemStartResp) - ahs := hostlist.MustCreateSet(tc.absentHosts) - resp.AbsentHosts.Replace(ahs) - ars := ranklist.MustCreateRankSet(tc.absentRanks) - resp.AbsentRanks.Replace(ars) - resp.Results = tc.results - test.CmpErr(t, tc.expErr, resp.Errors()) + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) }) } } @@ -850,11 +835,12 @@ func TestControl_SystemStop(t *testing.T) { testRespRS.AbsentRanks.Replace(testRS) for name, tc := range map[string]struct { - req *SystemStopReq - uErr error - uResp *UnaryResponse - expResp *SystemStopResp - expErr error + req *SystemStopReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemStopResp + expRespErr error }{ "nil req": { req: nil, @@ -872,53 +858,88 @@ func TestControl_SystemStop(t *testing.T) { }, "request absent host set": { req: testReqHS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemStopResp{ - Absenthosts: "foo-[1-23]", - }), - expResp: testRespHS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemStopResp{ + Absenthosts: "foo-[1-23]", + }), + expResp: testRespHS, + expRespErr: errors.New("non-existent hosts foo-[1-23]"), }, "request absent rank set": { req: testReqRS, - uResp: MockMSResponse("0.0.0.0", nil, - &mgmtpb.SystemStopResp{ - Absentranks: "1-23", - }), - expResp: testRespRS, + uResp: MockMSResponse("0.0.0.0", nil, &mgmtpb.SystemStopResp{ + Absentranks: "1-23", + }), + expResp: testRespRS, + expRespErr: errors.New("non-existent ranks 1-23"), }, - "dual host dual rank": { + "multiple member results": { req: new(SystemStopReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemStopResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 1, - State: system.MemberStateReady.String(), - }, - { - Rank: 2, - State: system.MemberStateReady.String(), - }, - { - Rank: 0, - State: system.MemberStateStopped.String(), - }, - { - Rank: 3, - State: system.MemberStateStopped.String(), - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemStopResp{ + Results: []*sharedpb.RankResult{ + { + Rank: 1, + State: system.MemberStateStopped.String(), + }, + { + Rank: 2, + State: system.MemberStateStopped.String(), + }, + { + Rank: 0, + State: system.MemberStateStopped.String(), + }, + { + Rank: 3, + State: system.MemberStateStopped.String(), }, }, - ), + }), expResp: &SystemStopResp{ Results: system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, nil, system.MemberStateReady), + system.NewMemberResult(1, nil, system.MemberStateStopped), + system.NewMemberResult(2, nil, system.MemberStateStopped), system.NewMemberResult(0, nil, system.MemberStateStopped), system.NewMemberResult(3, nil, system.MemberStateStopped), }, }, }, + "multiple member results; with errors": { + req: new(SystemStopReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemStopResp{ + Absenthosts: "foo-[1-23]", + Absentranks: "1-23", + Results: []*sharedpb.RankResult{ + { + Rank: 25, + State: system.MemberStateStopped.String(), + }, + { + Rank: 26, Errored: true, Msg: "fail", + State: system.MemberStateErrored.String(), + }, + { + Rank: 0, Errored: true, Msg: "failed", + State: system.MemberStateErrored.String(), + }, + { + Rank: 27, + State: system.MemberStateStopped.String(), + }, + }, + }), + expResp: &SystemStopResp{ + Results: system.MemberResults{ + system.NewMemberResult(25, nil, system.MemberStateStopped), + system.NewMemberResult(26, errors.New("fail"), + system.MemberStateStopped), + system.NewMemberResult(0, errors.New("failed"), + system.MemberStateStopped), + system.NewMemberResult(27, nil, system.MemberStateStopped), + }, + }, + expRespErr: errors.New("non-existent hosts foo-[1-23], " + + "non-existent ranks 1-23, check results for failed ranks 0,2"), + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) @@ -935,132 +956,285 @@ func TestControl_SystemStop(t *testing.T) { return } - cmpOpts := []cmp.Option{cmpopts.IgnoreUnexported(SystemStopResp{}, system.MemberResult{})} + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(SystemStopResp{}, system.MemberResult{}), + } if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } - if diff := cmp.Diff(tc.expResp.AbsentHosts.String(), gotResp.AbsentHosts.String()); diff != "" { - t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) + + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) + }) + } +} + +func TestControl_SystemExclude(t *testing.T) { + for name, tc := range map[string]struct { + req *SystemExcludeReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemExcludeResp + expRespErr error + }{ + "nil req": { + req: nil, + expErr: errors.New("nil *control.SystemExcludeReq request"), + }, + "local failure": { + req: new(SystemExcludeReq), + uErr: errors.New("local failed"), + expErr: errors.New("local failed"), + }, + "remote failure": { + req: new(SystemExcludeReq), + uResp: MockMSResponse("host1", errors.New("remote failed"), nil), + expErr: errors.New("remote failed"), + }, + "dual rank": { + req: new(SystemExcludeReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemExcludeResp{ + Results: []*sharedpb.RankResult{ + { + Rank: 1, + State: system.MemberStateAdminExcluded.String(), + }, + { + Rank: 0, + State: system.MemberStateAdminExcluded.String(), + }, + }, + }), + expResp: &SystemExcludeResp{ + Results: system.MemberResults{ + system.NewMemberResult(1, nil, system.MemberStateAdminExcluded), + system.NewMemberResult(0, nil, system.MemberStateAdminExcluded), + }, + }, + }, + "dual rank; with errors": { + req: new(SystemExcludeReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemExcludeResp{ + Results: []*sharedpb.RankResult{ + { + Rank: 24, Errored: true, Msg: "fail", + State: system.MemberStateErrored.String(), + }, + { + Rank: 0, Errored: true, Msg: "failed", + State: system.MemberStateErrored.String(), + }, + }, + }), + expResp: &SystemExcludeResp{ + Results: system.MemberResults{ + system.NewMemberResult(24, errors.New("fail"), + system.MemberStateStopped), + system.NewMemberResult(0, errors.New("failed"), + system.MemberStateStopped), + }, + }, + expRespErr: errors.New("failed ranks 0,24"), + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + mi := NewMockInvoker(log, &MockInvokerConfig{ + UnaryError: tc.uErr, + UnaryResponse: tc.uResp, + }) + + gotResp, gotErr := SystemExclude(test.Context(t), mi, tc.req) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return } - if diff := cmp.Diff(tc.expResp.AbsentRanks.String(), gotResp.AbsentRanks.String()); diff != "" { + + cmpOpts := []cmp.Option{cmpopts.IgnoreUnexported(SystemExcludeResp{}, + system.MemberResult{})} + if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } + + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) }) } } -func TestControl_SystemStopRespErrors(t *testing.T) { - successResults := system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, nil, system.MemberStateReady), - system.NewMemberResult(0, nil, system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), - } - failedResults := system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, errors.New("fail"), system.MemberStateReady), - system.NewMemberResult(0, errors.New("failed"), system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), - } - +func TestControl_SystemDrain(t *testing.T) { for name, tc := range map[string]struct { - absentHosts string - absentRanks string - results system.MemberResults - expErr error + req *SystemDrainReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemDrainResp + expRespErr error }{ - "no errors": { - results: successResults, + "nil req": { + req: nil, + expErr: errors.New("nil *control.SystemDrainReq request"), }, - "absent hosts": { - absentHosts: "foo-[1-23]", - results: successResults, - expErr: errors.New("non-existent hosts foo-[1-23]"), + "local failure": { + req: new(SystemDrainReq), + uErr: errors.New("local failed"), + expErr: errors.New("local failed"), }, - "absent ranks": { - absentRanks: "1-23", - results: successResults, - expErr: errors.New("non-existent ranks 1-23"), + "remote failure": { + req: new(SystemDrainReq), + uResp: MockMSResponse("host1", errors.New("remote failed"), nil), + expErr: errors.New("remote failed"), }, - "failed ranks": { - results: failedResults, - expErr: errors.New("check results for failed ranks 0,2"), + "dual pools; single rank": { + req: new(SystemDrainReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + {PoolId: test.MockUUID(1), Ranks: "1"}, + {PoolId: test.MockUUID(2), Ranks: "1"}, + }, + }), + expResp: &SystemDrainResp{ + Results: []*DrainResult{ + {PoolID: test.MockUUID(1), Ranks: "1"}, + {PoolID: test.MockUUID(2), Ranks: "1"}, + }, + }, }, - "absent hosts and ranks with failed ranks": { - absentHosts: "foo-[1-23]", - absentRanks: "1-23", - results: failedResults, - expErr: errors.New("non-existent hosts foo-[1-23], " + - "non-existent ranks 1-23, check results for failed ranks 0,2"), + "dual pools; single rank; with errors": { + req: new(SystemDrainReq), + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + { + PoolId: test.MockUUID(1), Ranks: "1", + Status: -1, Msg: "fail1", + }, + { + PoolId: test.MockUUID(2), Ranks: "1", + Status: -1, Msg: "fail2", + }, + }, + }), + expResp: &SystemDrainResp{ + Results: []*DrainResult{ + { + PoolID: test.MockUUID(1), Ranks: "1", + Status: -1, Msg: "fail1", + }, + { + PoolID: test.MockUUID(2), Ranks: "1", + Status: -1, Msg: "fail2", + }, + }, + }, + expRespErr: errors.New("fail1, fail2"), }, } { t.Run(name, func(t *testing.T) { - resp := new(SystemStopResp) - ahs := hostlist.MustCreateSet(tc.absentHosts) - resp.AbsentHosts.Replace(ahs) - ars := ranklist.MustCreateRankSet(tc.absentRanks) - resp.AbsentRanks.Replace(ars) - resp.Results = tc.results + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) - test.CmpErr(t, tc.expErr, resp.Errors()) + mi := NewMockInvoker(log, &MockInvokerConfig{ + UnaryError: tc.uErr, + UnaryResponse: tc.uResp, + }) + + gotResp, gotErr := SystemDrain(test.Context(t), mi, tc.req) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(SystemDrainResp{}), + } + if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { + t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) + } + + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) }) } } -func TestControl_SystemExclude(t *testing.T) { +func TestControl_SystemCleanup(t *testing.T) { for name, tc := range map[string]struct { - req *SystemExcludeReq - uErr error - uResp *UnaryResponse - expResp *SystemExcludeResp - expErr error + req *SystemCleanupReq + uErr error + uResp *UnaryResponse + expErr error + expResp *SystemCleanupResp + expRespErr error }{ "nil req": { req: nil, - expErr: errors.New("nil *control.SystemExcludeReq request"), + expErr: errors.New("nil *control.SystemCleanupReq request"), + }, + "missing machine name": { + req: new(SystemCleanupReq), + expErr: errors.New("requires a machine name"), }, "local failure": { - req: new(SystemExcludeReq), + req: &SystemCleanupReq{Machine: "foo"}, uErr: errors.New("local failed"), expErr: errors.New("local failed"), }, "remote failure": { - req: new(SystemExcludeReq), + req: &SystemCleanupReq{Machine: "foo"}, uResp: MockMSResponse("host1", errors.New("remote failed"), nil), expErr: errors.New("remote failed"), }, - "dual host dual rank": { - req: new(SystemExcludeReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemExcludeResp{ - Results: []*sharedpb.RankResult{ - { - Rank: 1, - State: system.MemberStateReady.String(), - }, - { - Rank: 2, - State: system.MemberStateReady.String(), - }, - { - Rank: 0, - State: system.MemberStateStopped.String(), - }, - { - Rank: 3, - State: system.MemberStateStopped.String(), - }, + "no cleanup": { + req: &SystemCleanupReq{Machine: "foo"}, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemCleanupResp{ + Results: []*mgmtpb.SystemCleanupResp_CleanupResult{}, + }), + expResp: &SystemCleanupResp{}, + }, + "cleanup multiple handles": { + req: &SystemCleanupReq{Machine: "foo"}, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemCleanupResp{ + Results: []*mgmtpb.SystemCleanupResp_CleanupResult{ + {PoolId: test.MockUUID(1), Count: 10}, + {PoolId: test.MockUUID(2), Count: 20}, + }, + }), + expResp: &SystemCleanupResp{ + Results: []*CleanupResult{ + {PoolID: test.MockUUID(1), Count: 10}, + {PoolID: test.MockUUID(2), Count: 20}, + }, + }, + }, + "cleanup multiple handles; with errors": { + req: &SystemCleanupReq{Machine: "foo"}, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemCleanupResp{ + Results: []*mgmtpb.SystemCleanupResp_CleanupResult{ + { + PoolId: test.MockUUID(1), + Status: -1, Msg: "fail1", }, + { + PoolId: test.MockUUID(3), + Status: -1, Msg: "fail3", + }, + {PoolId: test.MockUUID(2), Count: 20}, }, - ), - expResp: &SystemExcludeResp{ - Results: system.MemberResults{ - system.NewMemberResult(1, nil, system.MemberStateReady), - system.NewMemberResult(2, nil, system.MemberStateReady), - system.NewMemberResult(0, nil, system.MemberStateStopped), - system.NewMemberResult(3, nil, system.MemberStateStopped), + }), + expResp: &SystemCleanupResp{ + Results: []*CleanupResult{ + { + PoolID: test.MockUUID(1), + Status: -1, Msg: "fail1", + }, + { + PoolID: test.MockUUID(3), + Status: -1, Msg: "fail3", + }, + {PoolID: test.MockUUID(2), Count: 20}, }, }, + expRespErr: errors.New("fail1, fail3"), }, } { t.Run(name, func(t *testing.T) { @@ -1072,21 +1246,25 @@ func TestControl_SystemExclude(t *testing.T) { UnaryResponse: tc.uResp, }) - gotResp, gotErr := SystemExclude(test.Context(t), mi, tc.req) + gotResp, gotErr := SystemCleanup(test.Context(t), mi, tc.req) test.CmpErr(t, tc.expErr, gotErr) if tc.expErr != nil { return } - cmpOpts := []cmp.Option{cmpopts.IgnoreUnexported(SystemExcludeResp{}, system.MemberResult{})} + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(SystemCleanupResp{}), + } if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } + + test.CmpErr(t, tc.expRespErr, gotResp.Errors()) }) } } -func TestDmg_System_checkSystemErase(t *testing.T) { +func TestControl_System_checkSystemErase(t *testing.T) { for name, tc := range map[string]struct { uErr, expErr error members []*mgmtpb.SystemMember @@ -1185,22 +1363,20 @@ func TestControl_SystemErase(t *testing.T) { }, "single host dual rank": { req: new(SystemEraseReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemEraseResp{ - Results: []*sharedpb.RankResult{ - { - Rank: member1.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member1.Addr.String(), - }, - { - Rank: member2.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member2.Addr.String(), - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemEraseResp{ + Results: []*sharedpb.RankResult{ + { + Rank: member1.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member1.Addr.String(), + }, + { + Rank: member2.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member2.Addr.String(), }, }, - ), + }), expResp: &SystemEraseResp{ Results: system.MemberResults{ mockMemberResult(member1, "system erase", nil, system.MemberStateAwaitFormat), @@ -1210,23 +1386,21 @@ func TestControl_SystemErase(t *testing.T) { }, "single host dual rank one failed": { req: new(SystemEraseReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemEraseResp{ - Results: []*sharedpb.RankResult{ - { - Rank: member1.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member1.Addr.String(), - Errored: true, Msg: "erase failed", - }, - { - Rank: member2.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member2.Addr.String(), - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemEraseResp{ + Results: []*sharedpb.RankResult{ + { + Rank: member1.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member1.Addr.String(), + Errored: true, Msg: "erase failed", + }, + { + Rank: member2.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member2.Addr.String(), }, }, - ), + }), expResp: &SystemEraseResp{ HostErrorsResp: MockHostErrorsResp(t, &MockHostError{ @@ -1242,57 +1416,55 @@ func TestControl_SystemErase(t *testing.T) { }, "multiple hosts dual rank mixed results": { req: new(SystemEraseReq), - uResp: MockMSResponse("10.0.0.1:10001", nil, - &mgmtpb.SystemEraseResp{ - Results: []*sharedpb.RankResult{ - { - Rank: member1.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member1.Addr.String(), - Errored: true, Msg: "erase failed", - }, - { - Rank: member2.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member2.Addr.String(), - }, - { - Rank: member3.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member3.Addr.String(), - }, - { - Rank: member4.Rank.Uint32(), Action: "system erase", - State: system.MemberStateAwaitFormat.String(), - Addr: member4.Addr.String(), - }, - { - Rank: member5.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member5.Addr.String(), - Errored: true, Msg: "erase failed", - }, - { - Rank: member6.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member6.Addr.String(), - Errored: true, Msg: "something bad", - }, - { - Rank: member7.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member7.Addr.String(), - Errored: true, Msg: "erase failed", - }, - { - Rank: member8.Rank.Uint32(), Action: "system erase", - State: system.MemberStateErrored.String(), - Addr: member8.Addr.String(), - Errored: true, Msg: "erase failed", - }, + uResp: MockMSResponse("10.0.0.1:10001", nil, &mgmtpb.SystemEraseResp{ + Results: []*sharedpb.RankResult{ + { + Rank: member1.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member1.Addr.String(), + Errored: true, Msg: "erase failed", + }, + { + Rank: member2.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member2.Addr.String(), + }, + { + Rank: member3.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member3.Addr.String(), + }, + { + Rank: member4.Rank.Uint32(), Action: "system erase", + State: system.MemberStateAwaitFormat.String(), + Addr: member4.Addr.String(), + }, + { + Rank: member5.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member5.Addr.String(), + Errored: true, Msg: "erase failed", + }, + { + Rank: member6.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member6.Addr.String(), + Errored: true, Msg: "something bad", + }, + { + Rank: member7.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member7.Addr.String(), + Errored: true, Msg: "erase failed", + }, + { + Rank: member8.Rank.Uint32(), Action: "system erase", + State: system.MemberStateErrored.String(), + Addr: member8.Addr.String(), + Errored: true, Msg: "erase failed", }, }, - ), + }), expResp: &SystemEraseResp{ HostErrorsResp: MockHostErrorsResp(t, &MockHostError{ @@ -1345,6 +1517,90 @@ func TestControl_SystemErase(t *testing.T) { } } +func TestControl_LeaderQuery(t *testing.T) { + for name, tc := range map[string]struct { + req *LeaderQueryReq + uErr error + uResp *UnaryResponse + expResp *LeaderQueryResp + expErr error + }{ + "nil req": { + req: nil, + expErr: errors.New("nil *control.LeaderQueryReq request"), + }, + "local failure": { + req: new(LeaderQueryReq), + uErr: errors.New("local failed"), + expErr: errors.New("local failed"), + }, + "remote failure": { + req: new(LeaderQueryReq), + uResp: MockMSResponse("host1", errors.New("remote failed"), nil), + expErr: errors.New("remote failed"), + }, + "success": { + req: new(LeaderQueryReq), + uResp: MockMSResponse("host1", nil, &mgmtpb.LeaderQueryResp{}), + expResp: &LeaderQueryResp{}, + }, + "success; multiple replicas": { + req: new(LeaderQueryReq), + uResp: &UnaryResponse{ + Responses: []*HostResponse{ + { + Addr: "host1", + Message: &mgmtpb.LeaderQueryResp{}, + }, + { + Addr: "host2", + Message: &mgmtpb.LeaderQueryResp{}, + }, + }, + }, + expResp: &LeaderQueryResp{}, + }, + "replica failure": { + req: new(LeaderQueryReq), + uResp: &UnaryResponse{ + Responses: []*HostResponse{ + { + Addr: "host1", + Error: errors.New("remote failed"), + }, + { + Addr: "host2", + Message: &mgmtpb.LeaderQueryResp{}, + }, + }, + }, + expResp: &LeaderQueryResp{ + DownReplicas: []string{"host1"}, + }, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + mi := NewMockInvoker(log, &MockInvokerConfig{ + UnaryError: tc.uErr, + UnaryResponse: tc.uResp, + }) + + gotResp, gotErr := LeaderQuery(test.Context(t), mi, tc.req) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(tc.expResp, gotResp, defResCmpOpts()...); diff != "" { + t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) + } + }) + } +} + func TestControl_SystemJoin_RetryableErrors(t *testing.T) { for name, testErr := range map[string]error{ "system not formatted": system.ErrUninitialized, diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index c4ce6aed44c..d53cf439343 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -1156,7 +1156,7 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) daos.Status(drainResp.Status).Error()) } - svc.log.Tracef("pool-drain triggered from system-drain: '%+v' (req: '%+v')", + svc.log.Tracef("pool-drain triggered from system-drain: %+v (req: %+v)", drainResp, drainReq) resp.Results = append(resp.Results, &mgmtpb.SystemDrainResp_DrainResult{ From b635f8383f1b8379f22ed3a0fffa248797f63865 Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Thu, 21 Nov 2024 16:56:04 +0000 Subject: [PATCH 3/6] unexport pool command types Required-githooks: true Signed-off-by: Tom Nabarro --- src/control/cmd/dmg/pool.go | 114 +++++++++++++++---------------- src/control/cmd/dmg/pool_test.go | 10 +-- src/control/cmd/dmg/system.go | 2 +- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index 5ca56a18244..0b74128990b 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -30,23 +30,23 @@ import ( // PoolCmd is the struct representing the top-level pool subcommand. type PoolCmd struct { - Create PoolCreateCmd `command:"create" description:"Create a DAOS pool"` - Destroy PoolDestroyCmd `command:"destroy" description:"Destroy a DAOS pool"` - Evict PoolEvictCmd `command:"evict" description:"Evict all pool connections to a DAOS pool"` - List PoolListCmd `command:"list" alias:"ls" description:"List DAOS pools"` - Extend PoolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks."` - Exclude PoolExcludeCmd `command:"exclude" description:"Exclude targets from a rank"` - Drain PoolDrainCmd `command:"drain" description:"Drain targets from a rank"` - Reintegrate PoolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"` - Query PoolQueryCmd `command:"query" description:"Query a DAOS pool"` - QueryTargets PoolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"` - GetACL PoolGetACLCmd `command:"get-acl" description:"Get a DAOS pool's Access Control List"` - OverwriteACL PoolOverwriteACLCmd `command:"overwrite-acl" description:"Overwrite a DAOS pool's Access Control List"` - UpdateACL PoolUpdateACLCmd `command:"update-acl" description:"Update entries in a DAOS pool's Access Control List"` - DeleteACL PoolDeleteACLCmd `command:"delete-acl" description:"Delete an entry from a DAOS pool's Access Control List"` - SetProp PoolSetPropCmd `command:"set-prop" description:"Set pool property"` - GetProp PoolGetPropCmd `command:"get-prop" description:"Get pool properties"` - Upgrade PoolUpgradeCmd `command:"upgrade" description:"Upgrade pool to latest format"` + Create poolCreateCmd `command:"create" description:"Create a DAOS pool"` + Destroy poolDestroyCmd `command:"destroy" description:"Destroy a DAOS pool"` + Evict poolEvictCmd `command:"evict" description:"Evict all pool connections to a DAOS pool"` + List poolListCmd `command:"list" alias:"ls" description:"List DAOS pools"` + Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks."` + Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a rank"` + Drain poolDrainCmd `command:"drain" description:"Drain targets from a rank"` + Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"` + Query poolQueryCmd `command:"query" description:"Query a DAOS pool"` + QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"` + GetACL poolGetACLCmd `command:"get-acl" description:"Get a DAOS pool's Access Control List"` + OverwriteACL poolOverwriteACLCmd `command:"overwrite-acl" description:"Overwrite a DAOS pool's Access Control List"` + UpdateACL poolUpdateACLCmd `command:"update-acl" description:"Update entries in a DAOS pool's Access Control List"` + DeleteACL poolDeleteACLCmd `command:"delete-acl" description:"Delete an entry from a DAOS pool's Access Control List"` + SetProp poolSetPropCmd `command:"set-prop" description:"Set pool property"` + GetProp poolGetPropCmd `command:"get-prop" description:"Get pool properties"` + Upgrade poolUpgradeCmd `command:"upgrade" description:"Upgrade pool to latest format"` } var ( @@ -163,7 +163,7 @@ func (psf *poolSizeFlag) UnmarshalFlag(fv string) error { } // PoolCreateCmd is the struct representing the command to create a DAOS pool. -type PoolCreateCmd struct { +type poolCreateCmd struct { baseCmd cfgCmd ctlInvokerCmd @@ -207,7 +207,7 @@ func ratio2Percentage(log logging.Logger, scm, nvme float64) (p float64) { // MemRatio can be supplied as two fractions that make up 1 or a single fraction less than 1. // Supply only the first fraction in request and if not set then use the default. -func (cmd *PoolCreateCmd) setMemRatio(req *control.PoolCreateReq, defVal float32) error { +func (cmd *poolCreateCmd) setMemRatio(req *control.PoolCreateReq, defVal float32) error { if cmd.MemRatio.IsSet() { f, err := ratiosToSingleFraction(cmd.MemRatio.Ratios()) if err != nil { @@ -222,7 +222,7 @@ func (cmd *PoolCreateCmd) setMemRatio(req *control.PoolCreateReq, defVal float32 return nil } -func (cmd *PoolCreateCmd) storageAutoPercentage(ctx context.Context, req *control.PoolCreateReq) error { +func (cmd *poolCreateCmd) storageAutoPercentage(ctx context.Context, req *control.PoolCreateReq) error { if cmd.NumRanks > 0 { return errIncompatFlags("size", "nranks") } @@ -242,7 +242,7 @@ func (cmd *PoolCreateCmd) storageAutoPercentage(ctx context.Context, req *contro return nil } -func (cmd *PoolCreateCmd) storageAutoTotal(req *control.PoolCreateReq) error { +func (cmd *poolCreateCmd) storageAutoTotal(req *control.PoolCreateReq) error { if cmd.NumRanks > 0 && !cmd.RankList.Empty() { return errIncompatFlags("nranks", "ranks") } @@ -267,7 +267,7 @@ func (cmd *PoolCreateCmd) storageAutoTotal(req *control.PoolCreateReq) error { return nil } -func (cmd *PoolCreateCmd) storageManualMdOnSsd(req *control.PoolCreateReq) error { +func (cmd *poolCreateCmd) storageManualMdOnSsd(req *control.PoolCreateReq) error { metaBytes := cmd.MetaSize.Bytes dataBytes := cmd.DataSize.Bytes req.TierBytes = []uint64{metaBytes, dataBytes} @@ -288,7 +288,7 @@ func (cmd *PoolCreateCmd) storageManualMdOnSsd(req *control.PoolCreateReq) error return nil } -func (cmd *PoolCreateCmd) storageManual(req *control.PoolCreateReq) error { +func (cmd *poolCreateCmd) storageManual(req *control.PoolCreateReq) error { switch { case cmd.NumRanks > 0: return errIncompatFlags("nranks", "scm-size") @@ -317,7 +317,7 @@ func (cmd *PoolCreateCmd) storageManual(req *control.PoolCreateReq) error { } // Execute is run when PoolCreateCmd subcommand is activated -func (cmd *PoolCreateCmd) Execute(args []string) error { +func (cmd *poolCreateCmd) Execute(args []string) error { if cmd.Args.PoolLabel != "" { for _, prop := range cmd.Properties.ToSet { if prop.Name == "label" { @@ -402,7 +402,7 @@ func (cmd *PoolCreateCmd) Execute(args []string) error { } // PoolListCmd represents the command to fetch a list of all DAOS pools in the system. -type PoolListCmd struct { +type poolListCmd struct { baseCmd cfgCmd ctlInvokerCmd @@ -413,7 +413,7 @@ type PoolListCmd struct { } // Execute is run when PoolListCmd activates -func (cmd *PoolListCmd) Execute(_ []string) (errOut error) { +func (cmd *poolListCmd) Execute(_ []string) (errOut error) { defer func() { errOut = errors.Wrap(errOut, "list pools failed") }() @@ -482,14 +482,14 @@ func (cmd *poolCmd) PoolID() *PoolID { } // PoolDestroyCmd is the struct representing the command to destroy a DAOS pool. -type PoolDestroyCmd struct { +type poolDestroyCmd struct { poolCmd Recursive bool `short:"r" long:"recursive" description:"Remove pool with existing containers"` Force bool `short:"f" long:"force" description:"Forcibly remove pool with active client connections"` } // Execute is run when PoolDestroyCmd subcommand is activated -func (cmd *PoolDestroyCmd) Execute(args []string) error { +func (cmd *poolDestroyCmd) Execute(args []string) error { msg := "succeeded" req := &control.PoolDestroyReq{ @@ -510,12 +510,12 @@ func (cmd *PoolDestroyCmd) Execute(args []string) error { } // PoolEvictCmd is the struct representing the command to evict a DAOS pool. -type PoolEvictCmd struct { +type poolEvictCmd struct { poolCmd } // Execute is run when PoolEvictCmd subcommand is activated -func (cmd *PoolEvictCmd) Execute(args []string) error { +func (cmd *poolEvictCmd) Execute(args []string) error { msg := "succeeded" req := &control.PoolEvictReq{ID: cmd.PoolID().String()} @@ -531,14 +531,14 @@ func (cmd *PoolEvictCmd) Execute(args []string) error { } // PoolExcludeCmd is the struct representing the command to exclude a DAOS target. -type PoolExcludeCmd struct { +type poolExcludeCmd struct { poolCmd Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"` TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"` } // Execute is run when PoolExcludeCmd subcommand is activated -func (cmd *PoolExcludeCmd) Execute(args []string) error { +func (cmd *poolExcludeCmd) Execute(args []string) error { msg := "succeeded" var idxList []uint32 @@ -559,14 +559,14 @@ func (cmd *PoolExcludeCmd) Execute(args []string) error { } // PoolDrainCmd is the struct representing the command to Drain a DAOS target. -type PoolDrainCmd struct { +type poolDrainCmd struct { poolCmd Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be drained"` TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on the rank"` } // Execute is run when PoolDrainCmd subcommand is activated -func (cmd *PoolDrainCmd) Execute(args []string) error { +func (cmd *poolDrainCmd) Execute(args []string) error { msg := "succeeded" var idxList []uint32 @@ -592,13 +592,13 @@ func (cmd *PoolDrainCmd) Execute(args []string) error { } // PoolExtendCmd is the struct representing the command to Extend a DAOS pool. -type PoolExtendCmd struct { +type poolExtendCmd struct { poolCmd RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of ranks to add to the pool"` } // Execute is run when PoolExtendCmd subcommand is activated -func (cmd *PoolExtendCmd) Execute(args []string) error { +func (cmd *poolExtendCmd) Execute(args []string) error { msg := "succeeded" req := &control.PoolExtendReq{ @@ -617,14 +617,14 @@ func (cmd *PoolExtendCmd) Execute(args []string) error { } // PoolReintegrateCmd is the struct representing the command to Add a DAOS target. -type PoolReintegrateCmd struct { +type poolReintegrateCmd struct { poolCmd Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be reintegrated"` TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into the rank"` } // Execute is run when PoolReintegrateCmd subcommand is activated -func (cmd *PoolReintegrateCmd) Execute(args []string) error { +func (cmd *poolReintegrateCmd) Execute(args []string) error { msg := "succeeded" var idxList []uint32 @@ -650,14 +650,14 @@ func (cmd *PoolReintegrateCmd) Execute(args []string) error { } // PoolQueryCmd is the struct representing the command to query a DAOS pool. -type PoolQueryCmd struct { +type poolQueryCmd struct { poolCmd ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } // Execute is run when PoolQueryCmd subcommand is activated -func (cmd *PoolQueryCmd) Execute(args []string) error { +func (cmd *poolQueryCmd) Execute(args []string) error { req := &control.PoolQueryReq{ ID: cmd.PoolID().String(), QueryMask: daos.DefaultPoolQueryMask, @@ -695,7 +695,7 @@ func (cmd *PoolQueryCmd) Execute(args []string) error { } // PoolQueryTargetsCmd is the struct representing the command to query a DAOS pool engine's targets -type PoolQueryTargetsCmd struct { +type poolQueryTargetsCmd struct { poolCmd Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"` @@ -703,7 +703,7 @@ type PoolQueryTargetsCmd struct { } // Execute is run when PoolQueryTargetsCmd subcommand is activated -func (cmd *PoolQueryTargetsCmd) Execute(args []string) error { +func (cmd *poolQueryTargetsCmd) Execute(args []string) error { ctx := cmd.MustLogCtx() var tgtsList []uint32 @@ -753,12 +753,12 @@ func (cmd *PoolQueryTargetsCmd) Execute(args []string) error { } // PoolUpgradeCmd is the struct representing the command to update a DAOS pool. -type PoolUpgradeCmd struct { +type poolUpgradeCmd struct { poolCmd } // Execute is run when PoolUpgradeCmd subcommand is activated -func (cmd *PoolUpgradeCmd) Execute(args []string) error { +func (cmd *poolUpgradeCmd) Execute(args []string) error { req := &control.PoolUpgradeReq{ ID: cmd.PoolID().String(), } @@ -773,7 +773,7 @@ func (cmd *PoolUpgradeCmd) Execute(args []string) error { } // PoolSetPropCmd represents the command to set a property on a pool. -type PoolSetPropCmd struct { +type poolSetPropCmd struct { poolCmd Args struct { @@ -782,7 +782,7 @@ type PoolSetPropCmd struct { } // Execute is run when PoolSetPropCmd subcommand is activatecmd. -func (cmd *PoolSetPropCmd) Execute(_ []string) error { +func (cmd *poolSetPropCmd) Execute(_ []string) error { for _, prop := range cmd.Args.Props.ToSet { if prop.Name == "perf_domain" { return errors.New("can't set perf_domain on existing pool.") @@ -817,7 +817,7 @@ func (cmd *PoolSetPropCmd) Execute(_ []string) error { } // PoolGetPropCmd represents the command to set a property on a pool. -type PoolGetPropCmd struct { +type poolGetPropCmd struct { poolCmd Args struct { Props PoolGetPropsFlag `positional-arg-name:"[key[,key...]]"` @@ -825,7 +825,7 @@ type PoolGetPropCmd struct { } // Execute is run when PoolGetPropCmd subcommand is activatecmd. -func (cmd *PoolGetPropCmd) Execute(_ []string) error { +func (cmd *poolGetPropCmd) Execute(_ []string) error { req := &control.PoolGetPropReq{ ID: cmd.PoolID().String(), Properties: cmd.Args.Props.ToGet, @@ -849,7 +849,7 @@ func (cmd *PoolGetPropCmd) Execute(_ []string) error { // PoolGetACLCmd represents the command to fetch an Access Control List of a // DAOS pool. -type PoolGetACLCmd struct { +type poolGetACLCmd struct { poolCmd File string `short:"o" long:"outfile" required:"0" description:"Output ACL to file"` Force bool `short:"f" long:"force" required:"0" description:"Allow to clobber output file"` @@ -857,7 +857,7 @@ type PoolGetACLCmd struct { } // Execute is run when the PoolGetACLCmd subcommand is activated -func (cmd *PoolGetACLCmd) Execute(args []string) error { +func (cmd *poolGetACLCmd) Execute(args []string) error { req := &control.PoolGetACLReq{ID: cmd.PoolID().String()} resp, err := control.PoolGetACL(cmd.MustLogCtx(), cmd.ctlInvoker, req) @@ -886,7 +886,7 @@ func (cmd *PoolGetACLCmd) Execute(args []string) error { return nil } -func (cmd *PoolGetACLCmd) writeACLToFile(acl string) error { +func (cmd *poolGetACLCmd) writeACLToFile(acl string) error { if !cmd.Force { // Keep the user from clobbering existing files _, err := os.Stat(cmd.File) @@ -913,13 +913,13 @@ func (cmd *PoolGetACLCmd) writeACLToFile(acl string) error { // PoolOverwriteACLCmd represents the command to overwrite the Access Control // List of a DAOS pool. -type PoolOverwriteACLCmd struct { +type poolOverwriteACLCmd struct { poolCmd ACLFile string `short:"a" long:"acl-file" required:"1" description:"Path for new Access Control List file"` } // Execute is run when the PoolOverwriteACLCmd subcommand is activated -func (cmd *PoolOverwriteACLCmd) Execute(args []string) error { +func (cmd *poolOverwriteACLCmd) Execute(args []string) error { acl, err := control.ReadACLFile(cmd.ACLFile) if err != nil { return err @@ -948,14 +948,14 @@ func (cmd *PoolOverwriteACLCmd) Execute(args []string) error { // PoolUpdateACLCmd represents the command to update the Access Control List of // a DAOS pool. -type PoolUpdateACLCmd struct { +type poolUpdateACLCmd struct { poolCmd ACLFile string `short:"a" long:"acl-file" required:"0" description:"Path for new Access Control List file"` Entry string `short:"e" long:"entry" required:"0" description:"Single Access Control Entry to add or update"` } // Execute is run when the PoolUpdateACLCmd subcommand is activated -func (cmd *PoolUpdateACLCmd) Execute(args []string) error { +func (cmd *poolUpdateACLCmd) Execute(args []string) error { if (cmd.ACLFile == "" && cmd.Entry == "") || (cmd.ACLFile != "" && cmd.Entry != "") { return errors.New("either ACL file or entry parameter is required") } @@ -996,13 +996,13 @@ func (cmd *PoolUpdateACLCmd) Execute(args []string) error { // PoolDeleteACLCmd represents the command to delete an entry from the Access // Control List of a DAOS pool. -type PoolDeleteACLCmd struct { +type poolDeleteACLCmd struct { poolCmd Principal string `short:"p" long:"principal" required:"1" description:"Principal whose entry should be removed"` } // Execute is run when the PoolDeleteACLCmd subcommand is activated -func (cmd *PoolDeleteACLCmd) Execute(args []string) error { +func (cmd *poolDeleteACLCmd) Execute(args []string) error { req := &control.PoolDeleteACLReq{ ID: cmd.PoolID().String(), Principal: cmd.Principal, diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index 5d30ec2dfb1..4681abd9f17 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -1272,12 +1272,12 @@ func TestDmg_PoolListCmd_Errors(t *testing.T) { UnaryResponseSet: responses, }) - PoolListCmd := new(PoolListCmd) - PoolListCmd.setInvoker(mi) - PoolListCmd.SetLog(log) - PoolListCmd.setConfig(tc.ctlCfg) + cmd := new(poolListCmd) + cmd.setInvoker(mi) + cmd.SetLog(log) + cmd.setConfig(tc.ctlCfg) - gotErr := PoolListCmd.Execute(nil) + gotErr := cmd.Execute(nil) test.CmpErr(t, tc.expErr, gotErr) }) } diff --git a/src/control/cmd/dmg/system.go b/src/control/cmd/dmg/system.go index ee5213934d9..86a4182f863 100644 --- a/src/control/cmd/dmg/system.go +++ b/src/control/cmd/dmg/system.go @@ -35,7 +35,7 @@ type SystemCmd struct { ClearExclude systemClearExcludeCmd `command:"clear-exclude" description:"Clear excluded state for ranks"` Drain systemDrainCmd `command:"drain" description:"Drain ranks or hosts from all relevant pools in DAOS system"` Erase systemEraseCmd `command:"erase" description:"Erase system metadata prior to reformat"` - ListPools PoolListCmd `command:"list-pools" description:"List all pools in the DAOS system"` + ListPools poolListCmd `command:"list-pools" description:"List all pools in the DAOS system"` Cleanup systemCleanupCmd `command:"cleanup" description:"Clean up all resources associated with the specified machine"` SetAttr systemSetAttrCmd `command:"set-attr" description:"Set system attributes"` GetAttr systemGetAttrCmd `command:"get-attr" description:"Get system attributes"` From 77340304764ebd9a4978c6a2bef8dbd7c7b372cf Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Fri, 22 Nov 2024 22:14:37 +0000 Subject: [PATCH 4/6] add unit test coverage for system drain in mgmt_system_test.go Features: control Required-githooks: true Signed-off-by: Tom Nabarro --- src/control/server/mgmt_system.go | 27 ++-- src/control/server/mgmt_system_test.go | 216 +++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 9 deletions(-) diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index d53cf439343..9c140122875 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -14,6 +14,7 @@ import ( "path/filepath" "reflect" "runtime" + "sort" "strconv" "strings" "time" @@ -1113,20 +1114,30 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) } poolRanks := make(map[string]*ranklist.RankSet) + poolIDs := []string{} for _, ps := range psList { - for _, r := range ps.Storage.CurrentRanks() { + currentRanks := ps.Storage.CurrentRanks() + poolID := ps.PoolUUID.String() + + svc.log.Tracef("pool-service detected: id %s, ranks %v", poolID, currentRanks) + + for _, r := range currentRanks { if _, exists := ranksMap[r]; !exists { continue } - id := ps.PoolUUID.String() - if _, exists := poolRanks[id]; !exists { - poolRanks[id] = ranklist.MustCreateRankSet("") + if _, exists := poolRanks[poolID]; !exists { + poolRanks[poolID] = ranklist.MustCreateRankSet("") + poolIDs = append(poolIDs, poolID) } - poolRanks[id].Add(r) + poolRanks[poolID].Add(r) } } + svc.log.Debugf("pool-ranks to drain: %v", poolRanks) + + sort.Strings(poolIDs) - for id, rs := range poolRanks { + for _, id := range poolIDs { + rs := poolRanks[id] if rs.Count() == 0 { continue } @@ -1151,9 +1162,7 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) errMsg = errors.Wrap(err, "unmarshal PoolEvict response").Error() drainResp.Status = int32(daos.IOInvalid) } else if drainResp.Status != int32(daos.Success) { - errMsg = fmt.Sprintf("drain failed for rank %d on pool %s: %s", - drainReq.Rank, drainReq.Id, - daos.Status(drainResp.Status).Error()) + errMsg = daos.Status(drainResp.Status).Error() } svc.log.Tracef("pool-drain triggered from system-drain: %+v (req: %+v)", diff --git a/src/control/server/mgmt_system_test.go b/src/control/server/mgmt_system_test.go index 0095cd3c87f..67888cba981 100644 --- a/src/control/server/mgmt_system_test.go +++ b/src/control/server/mgmt_system_test.go @@ -18,6 +18,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + uuid "github.com/google/uuid" "github.com/pkg/errors" "google.golang.org/grpc/peer" "google.golang.org/protobuf/proto" @@ -1829,6 +1830,221 @@ func TestServer_MgmtSvc_SystemExclude(t *testing.T) { } } +func TestServer_MgmtSvc_SystemDrain(t *testing.T) { + dReq := func(id, rank int) *mgmtpb.PoolDrainReq { + return &mgmtpb.PoolDrainReq{ + Sys: "daos_server", + Id: test.MockUUID(int32(id)), + Rank: uint32(rank), + SvcRanks: []uint32{0}, + } + } + + for name, tc := range map[string]struct { + members system.Members + req *mgmtpb.SystemDrainReq + expDrpcReqs []*mgmt.PoolDrainReq + drpcResp *mgmtpb.PoolDrainResp + drpcErr error + poolRanks map[string]string + expResp *mgmtpb.SystemDrainResp + expErr error + }{ + "nil req": { + req: (*mgmtpb.SystemDrainReq)(nil), + expErr: errors.New("nil request"), + }, + "not system leader": { + req: &mgmtpb.SystemDrainReq{ + Sys: "quack", + }, + expErr: FaultWrongSystem("quack", build.DefaultSystemName), + }, + "no hosts or ranks": { + req: &mgmtpb.SystemDrainReq{}, + expErr: errors.New("no hosts or ranks"), + }, + "hosts and ranks": { + req: &mgmtpb.SystemDrainReq{ + Hosts: "host1,host2", + Ranks: "0,1", + }, + expErr: errors.New("ranklist and hostlist"), + }, + "invalid ranks": { + req: &mgmtpb.SystemDrainReq{Ranks: "41,42"}, + expErr: errors.New("invalid rank(s)"), + }, + "invalid hosts": { + req: &mgmtpb.SystemDrainReq{Hosts: "host-[1-2]"}, + expErr: errors.New("invalid host(s)"), + }, + "no matching ranks": { + req: &mgmtpb.SystemDrainReq{Ranks: "0,1"}, + poolRanks: map[string]string{ + test.MockUUID(1): "2-5", + }, + expResp: &mgmtpb.SystemDrainResp{}, + }, + "matching ranks; multiple pools; no drpc response": { + req: &mgmtpb.SystemDrainReq{Ranks: "0,1"}, + poolRanks: map[string]string{ + test.MockUUID(1): "0-4", + test.MockUUID(2): "1-7", + }, + expErr: errors.New("not responding on dRPC"), + }, + "matching ranks; multiple pools": { + req: &mgmtpb.SystemDrainReq{Ranks: "0,1"}, + poolRanks: map[string]string{ + test.MockUUID(1): "0-4", + test.MockUUID(2): "1-7", + }, + drpcResp: &mgmtpb.PoolDrainResp{}, + expDrpcReqs: []*mgmtpb.PoolDrainReq{ + dReq(1, 0), dReq(1, 1), dReq(2, 1), + }, + expResp: &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + {PoolId: test.MockUUID(1), Ranks: "0"}, + {PoolId: test.MockUUID(1), Ranks: "1"}, + {PoolId: test.MockUUID(2), Ranks: "1"}, + }, + }, + }, + "matching hosts; multiple pools": { + req: &mgmtpb.SystemDrainReq{ + // Resolves to ranks 0-3. + Hosts: fmt.Sprintf("%s,%s", test.MockHostAddr(1), + test.MockHostAddr(2)), + }, + poolRanks: map[string]string{ + test.MockUUID(1): "0-4", + test.MockUUID(2): "1-7", + }, + drpcResp: &mgmtpb.PoolDrainResp{}, + expDrpcReqs: []*mgmtpb.PoolDrainReq{ + dReq(1, 0), dReq(1, 1), dReq(1, 2), dReq(1, 3), + dReq(2, 1), dReq(2, 2), dReq(2, 3), + }, + expResp: &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + {PoolId: test.MockUUID(1), Ranks: "0"}, + {PoolId: test.MockUUID(1), Ranks: "1"}, + {PoolId: test.MockUUID(1), Ranks: "2"}, + {PoolId: test.MockUUID(1), Ranks: "3"}, + {PoolId: test.MockUUID(2), Ranks: "1"}, + {PoolId: test.MockUUID(2), Ranks: "2"}, + {PoolId: test.MockUUID(2), Ranks: "3"}, + }, + }, + }, + "matching ranks; variable states; drpc fails": { + members: system.Members{ + mockMember(t, 2, 0, "errored"), + mockMember(t, 1, 0, "excluded"), + }, + req: &mgmtpb.SystemDrainReq{Ranks: "1-2"}, + poolRanks: map[string]string{test.MockUUID(1): "1-2"}, + drpcResp: &mgmtpb.PoolDrainResp{Status: -1}, + expDrpcReqs: []*mgmtpb.PoolDrainReq{ + dReq(1, 1), dReq(1, 2), + }, + expResp: &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + { + PoolId: test.MockUUID(1), + Ranks: "1", + Status: -1, + Msg: "DER_UNKNOWN(-1): Unknown error code -1", + }, + { + PoolId: test.MockUUID(1), + Ranks: "2", + Status: -1, + Msg: "DER_UNKNOWN(-1): Unknown error code -1", + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + if tc.members == nil { + tc.members = system.Members{ + mockMember(t, 0, 1, "joined"), + mockMember(t, 1, 2, "joined"), + mockMember(t, 2, 2, "joined"), + mockMember(t, 3, 1, "joined"), + mockMember(t, 4, 3, "joined"), + mockMember(t, 5, 3, "joined"), + mockMember(t, 6, 4, "joined"), + mockMember(t, 7, 4, "joined"), + } + } + svc := mgmtSystemTestSetup(t, log, tc.members, nil) + + for uuidStr, ranksStr := range tc.poolRanks { + addTestPoolService(t, svc.sysdb, &system.PoolService{ + PoolUUID: uuid.MustParse(uuidStr), + State: system.PoolServiceStateReady, + Storage: &system.PoolServiceStorage{ + CurrentRankStr: ranksStr, + }, + Replicas: []ranklist.Rank{0}, + }) + } + + var mockDrpc *mockDrpcClient + if tc.drpcResp != nil { + mockDrpc = getMockDrpcClient(tc.drpcResp, tc.drpcErr) + setupSvcDrpcClient(svc, 0, mockDrpc) + } + + if tc.req != nil && tc.req.Sys == "" { + tc.req.Sys = build.DefaultSystemName + } + + gotResp, gotErr := svc.SystemDrain(test.MustLogContext(t, log), tc.req) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(mgmtpb.SystemDrainResp{}, + mgmtpb.SystemDrainResp_DrainResult{}), + } + if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { + t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) + } + + if mockDrpc == nil { + return + } + + gotDrpcCalls := mockDrpc.calls.get() + test.AssertEqual(t, len(tc.expDrpcReqs), len(gotDrpcCalls), + "unexpected number of drpc calls") + + for i := range gotDrpcCalls { + gotReq := new(mgmtpb.PoolDrainReq) + err := proto.Unmarshal(gotDrpcCalls[i].Body, gotReq) + if err != nil { + t.Fatal(err) + } + opt := cmpopts.IgnoreUnexported(mgmtpb.PoolDrainReq{}) + diff := cmp.Diff(tc.expDrpcReqs[i], gotReq, opt) + if diff != "" { + t.Fatalf("want-, got+:\n%s", diff) + } + } + }) + } +} + func TestServer_MgmtSvc_SystemErase(t *testing.T) { hr := func(a int32, rrs ...*sharedpb.RankResult) *control.HostResponse { return &control.HostResponse{ From db500582c0febb349953c9315763b9a7eec5839c Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Tue, 26 Nov 2024 11:59:19 +0000 Subject: [PATCH 5/6] simplify pretty printing for drain and cleanup, improve error reporting and use labels over uuid when available Features: control Required-githooks: true Signed-off-by: Tom Nabarro --- src/control/cmd/dmg/pretty/system.go | 75 ++++++------------- src/control/cmd/dmg/pretty/system_test.go | 83 ++++++++++++++++++++-- src/control/cmd/dmg/system.go | 34 +++------ src/control/cmd/dmg/system_test.go | 16 ++++- src/control/common/proto/mgmt/system.pb.go | 8 +-- src/control/lib/control/system.go | 3 +- src/control/lib/control/system_test.go | 2 +- src/control/server/mgmt_system.go | 55 +++++++++++--- src/control/server/mgmt_system_test.go | 62 +++++++++++----- src/proto/mgmt/system.proto | 6 +- 10 files changed, 224 insertions(+), 120 deletions(-) diff --git a/src/control/cmd/dmg/pretty/system.go b/src/control/cmd/dmg/pretty/system.go index 8f7c7296cc2..aa0f0cce546 100644 --- a/src/control/cmd/dmg/pretty/system.go +++ b/src/control/cmd/dmg/pretty/system.go @@ -185,10 +185,10 @@ func PrintSystemStopResponse(out, outErr io.Writer, resp *control.SystemStopResp return printSystemResults(out, outErr, resp.Results, &resp.AbsentHosts, &resp.AbsentRanks) } -func printSystemCleanupRespVerbose(out io.Writer, resp *control.SystemCleanupResp) error { +func printSystemCleanupRespVerbose(out io.Writer, resp *control.SystemCleanupResp) { if len(resp.Results) == 0 { fmt.Fprintln(out, "no handles cleaned up") - return nil + return } titles := []string{"Pool", "Handles Revoked"} @@ -204,81 +204,52 @@ func printSystemCleanupRespVerbose(out io.Writer, resp *control.SystemCleanupRes } fmt.Fprintln(out, formatter.Format(table)) - - return nil } // PrintSystemCleanupResponse generates a human-readable representation of the // supplied SystemCleanupResp struct and writes it to the supplied io.Writer. -func PrintSystemCleanupResponse(out, outErr io.Writer, resp *control.SystemCleanupResp, verbose bool) error { - err := resp.Errors() - - if err != nil { - fmt.Fprintln(outErr, err.Error()) - } - +func PrintSystemCleanupResponse(out io.Writer, resp *control.SystemCleanupResp, verbose bool) { if len(resp.Results) == 0 { fmt.Fprintln(out, "No handles cleaned up") - return nil + return } if verbose { - return printSystemCleanupRespVerbose(out, resp) + printSystemCleanupRespVerbose(out, resp) + return } fmt.Fprintln(out, "System Cleanup Success") - return nil } -func printSystemDrainRespVerbose(out io.Writer, resp *control.SystemDrainResp) error { +// PrintSystemDrainResponse generates a human-readable representation of the supplied +// SystemDrainResp struct and writes it to the supplied io.Writer. Result related errors written to +// error io.Writer. +func PrintSystemDrainResponse(out io.Writer, resp *control.SystemDrainResp) { if len(resp.Results) == 0 { - fmt.Fprintln(out, "no pool ranks drained") - return nil + fmt.Fprintln(out, "No pool ranks drained") + return } - titles := []string{"Pool", "Ranks Drained"} + titles := []string{"Pool", "Ranks", "Result", "Reason"} formatter := txtfmt.NewTableFormatter(titles...) var table []txtfmt.TableRow for _, r := range resp.Results { + result := "OK" + reason := "N/A" + if r.Status != 0 { + result = "Failed" + reason = r.Msg + } row := txtfmt.TableRow{ - "Pool": r.PoolID, - "Ranks Drained": r.Ranks, + "Pool": r.PoolID, + "Ranks": r.Ranks, + "Result": result, + "Reason": reason, } table = append(table, row) } fmt.Fprintln(out, formatter.Format(table)) - - return nil -} - -// PrintSystemDrainResponse generates a human-readable representation of the -// supplied SystemDrainResp struct and writes it to the supplied io.Writer. -func PrintSystemDrainResponse(out, outErr io.Writer, resp *control.SystemDrainResp, verbose bool) error { - err := resp.Errors() - - if err != nil { - fmt.Fprintln(outErr, err.Error()) - } - - if len(resp.Results) == 0 { - fmt.Fprintln(out, "No pool ranks drained") - return nil - } - - if verbose { - return printSystemDrainRespVerbose(out, resp) - } - - var msg string - for i, result := range resp.Results { - if i != 0 { - fmt.Sprintf("%s, ", msg) - } - fmt.Sprintf("%s%s: %s", msg, result.PoolID, result.Ranks) - } - - fmt.Fprintf(out, "System Drain Success. Drained pool ranks: %s\n", msg) - return nil } diff --git a/src/control/cmd/dmg/pretty/system_test.go b/src/control/cmd/dmg/pretty/system_test.go index b772d1591c5..bfc33bb7c33 100644 --- a/src/control/cmd/dmg/pretty/system_test.go +++ b/src/control/cmd/dmg/pretty/system_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -103,7 +103,7 @@ Ranks Action Result } if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { - t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + t.Fatalf("unexpected string output (-want, +got):\n%s\n", diff) } }) } @@ -357,7 +357,7 @@ Unknown 3 ranks: 7-9 } if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { - t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + t.Fatalf("unexpected string output (-want, +got):\n%s\n", diff) } }) } @@ -451,7 +451,7 @@ Unknown 3 hosts: foo[7-9] } if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { - t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + t.Fatalf("unexpected string output (-want, +got):\n%s\n", diff) } }) } @@ -606,7 +606,80 @@ Unknown 3 hosts: foo[7-9] } if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { - t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + t.Fatalf("unexpected string output (-want, +got):\n%s\n", diff) + } + }) + } +} + +func TestPretty_PrintSystemDrainResp(t *testing.T) { + for name, tc := range map[string]struct { + resp *control.SystemDrainResp + expOut string + }{ + "empty response": { + resp: &control.SystemDrainResp{}, + expOut: ` +No pool ranks drained +`, + }, + "normal response": { + resp: &control.SystemDrainResp{ + Results: []*control.DrainResult{ + {PoolID: test.MockUUID(1), Ranks: "0-3"}, + {PoolID: test.MockUUID(2), Ranks: "1-4"}, + }, + }, + expOut: ` +Pool Ranks Result Reason +---- ----- ------ ------ +00000001-0001-0001-0001-000000000001 0-3 OK N/A +00000002-0002-0002-0002-000000000002 1-4 OK N/A + +`, + }, + "normal response; use labels": { + resp: &control.SystemDrainResp{ + Results: []*control.DrainResult{ + {PoolID: "label1", Ranks: "0-3"}, + {PoolID: "label2", Ranks: "1-4"}, + }, + }, + expOut: ` +Pool Ranks Result Reason +---- ----- ------ ------ +label1 0-3 OK N/A +label2 1-4 OK N/A + +`, + }, + "response with failures": { + resp: &control.SystemDrainResp{ + Results: []*control.DrainResult{ + {PoolID: test.MockUUID(1), Ranks: "1-2"}, + {PoolID: test.MockUUID(2), Ranks: "0"}, + { + PoolID: test.MockUUID(2), Ranks: "1-2", + Status: -1, Msg: "fail1", + }, + }, + }, + expOut: ` +Pool Ranks Result Reason +---- ----- ------ ------ +00000001-0001-0001-0001-000000000001 1-2 OK N/A +00000002-0002-0002-0002-000000000002 0 OK N/A +00000002-0002-0002-0002-000000000002 1-2 Failed fail1 + +`, + }, + } { + t.Run(name, func(t *testing.T) { + var out strings.Builder + PrintSystemDrainResponse(&out, tc.resp) + + if diff := cmp.Diff(strings.TrimLeft(tc.expOut, "\n"), out.String()); diff != "" { + t.Fatalf("unexpected stdout (-want, +got):\n%s\n", diff) } }) } diff --git a/src/control/cmd/dmg/system.go b/src/control/cmd/dmg/system.go index 86a4182f863..a507a05004f 100644 --- a/src/control/cmd/dmg/system.go +++ b/src/control/cmd/dmg/system.go @@ -289,7 +289,7 @@ func (cmd *systemExcludeCmd) execute(clear bool) error { } cmd.Infof("updated ranks: %s", updated) - return nil + return resp.Errors() } func (cmd *systemExcludeCmd) Execute(_ []string) error { @@ -306,7 +306,6 @@ func (cmd *systemClearExcludeCmd) Execute(_ []string) error { type systemDrainCmd struct { baseRankListCmd - Verbose bool `long:"verbose" short:"v" description:"Output additional system drain information"` } func (cmd *systemDrainCmd) Execute(_ []string) (errOut error) { @@ -335,19 +334,11 @@ func (cmd *systemDrainCmd) Execute(_ []string) (errOut error) { return cmd.OutputJSON(resp, resp.Errors()) } - var out, outErr strings.Builder - if err := pretty.PrintSystemDrainResponse(&out, &outErr, resp, cmd.Verbose); err != nil { - return err - } - if outErr.String() != "" { - cmd.Error(outErr.String()) - } - - // Infof prints raw string and doesn't try to expand "%" - // preserving column formatting in txtfmt table - cmd.Infof("%s", out.String()) + var out strings.Builder + pretty.PrintSystemDrainResponse(&out, resp) + cmd.Info(out.String()) - return nil + return resp.Errors() } type systemCleanupCmd struct { @@ -376,17 +367,14 @@ func (cmd *systemCleanupCmd) Execute(_ []string) (errOut error) { return cmd.OutputJSON(resp, resp.Errors()) } - var out, outErr strings.Builder - if err := pretty.PrintSystemCleanupResponse(&out, &outErr, resp, cmd.Verbose); err != nil { - return err - } - if outErr.String() != "" { - cmd.Error(outErr.String()) + var out strings.Builder + pretty.PrintSystemCleanupResponse(&out, resp, cmd.Verbose) + + if resp.Errors() != nil { + cmd.Error(resp.Errors().Error()) } - // Infof prints raw string and doesn't try to expand "%" - // preserving column formatting in txtfmt table - cmd.Infof("%s", out.String()) + cmd.Info(out.String()) return resp.Errors() } diff --git a/src/control/cmd/dmg/system_test.go b/src/control/cmd/dmg/system_test.go index a1cc47c2c0e..18ca14f35e2 100644 --- a/src/control/cmd/dmg/system_test.go +++ b/src/control/cmd/dmg/system_test.go @@ -258,9 +258,19 @@ func TestDmg_SystemCommands(t *testing.T) { "", errNoRanks, }, + { + "system drain with multiple hosts", + "system drain --rank-hosts foo-[0,1,4]", + strings.Join([]string{ + printRequest(t, withSystem( + withHosts(&control.SystemDrainReq{}, "foo-[0-1,4]"), + "daos_server")), + }, " "), + nil, + }, { "system drain with multiple ranks", - "system drain --ranks 0,1,4 -v", + "system drain --ranks 0,1,4", strings.Join([]string{ printRequest(t, withSystem( withRanks(&control.SystemDrainReq{}, 0, 1, 4), @@ -270,13 +280,13 @@ func TestDmg_SystemCommands(t *testing.T) { }, { "system drain without ranks", - "system drain -v", + "system drain", "", errNoRanks, }, { "system cleanup with machine name", - "system cleanup foo1 -v", + "system cleanup foo1", strings.Join([]string{ printRequest(t, withSystem(&control.SystemCleanupReq{ Machine: "foo1", diff --git a/src/control/common/proto/mgmt/system.pb.go b/src/control/common/proto/mgmt/system.pb.go index afddaa4fc22..186b22c91c4 100644 --- a/src/control/common/proto/mgmt/system.pb.go +++ b/src/control/common/proto/mgmt/system.pb.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -637,7 +637,7 @@ type SystemDrainResp struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*SystemDrainResp_DrainResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Results for individual pool-ranks drain calls. + Results []*SystemDrainResp_DrainResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // Results for pool-ranks drain calls. } func (x *SystemDrainResp) Reset() { @@ -1359,8 +1359,8 @@ type SystemDrainResp_DrainResult struct { Status int32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // Status of the evict on the specific pool Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"` // Error message if status indicates an error - PoolId string `protobuf:"bytes,3,opt,name=pool_id,json=poolId,proto3" json:"pool_id,omitempty"` // uuid of pool - Ranks string `protobuf:"bytes,4,opt,name=ranks,proto3" json:"ranks,omitempty"` // rankset to have drained on poolexclude + PoolId string `protobuf:"bytes,3,opt,name=pool_id,json=poolId,proto3" json:"pool_id,omitempty"` // Label or uuid of pool + Ranks string `protobuf:"bytes,4,opt,name=ranks,proto3" json:"ranks,omitempty"` // Rank-set that has encountered this result } func (x *SystemDrainResp_DrainResult) Reset() { diff --git a/src/control/lib/control/system.go b/src/control/lib/control/system.go index d87c2df3b25..d2d64c5733e 100644 --- a/src/control/lib/control/system.go +++ b/src/control/lib/control/system.go @@ -595,7 +595,8 @@ type SystemDrainResp struct { func (sdr *SystemDrainResp) Errors() (errOut error) { for _, r := range sdr.Results { if r.Status != int32(daos.Success) { - errOut = concatErrs(errOut, errors.New(r.Msg)) + errOut = concatErrs(errOut, + errors.Errorf("pool %s ranks %s: %s", r.PoolID, r.Ranks, r.Msg)) } } diff --git a/src/control/lib/control/system_test.go b/src/control/lib/control/system_test.go index 6f1d43fe429..f4abbfc7b05 100644 --- a/src/control/lib/control/system_test.go +++ b/src/control/lib/control/system_test.go @@ -1127,7 +1127,7 @@ func TestControl_SystemDrain(t *testing.T) { }, }, }, - expRespErr: errors.New("fail1, fail2"), + expRespErr: errors.New("pool 00000001-0001-0001-0001-000000000001 ranks 1: fail1, pool 00000002-0002-0002-0002-000000000002 ranks 1: fail2"), }, } { t.Run(name, func(t *testing.T) { diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index 9c140122875..4dab1724b37 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -1104,20 +1104,21 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) return nil, err } - resp := new(mgmtpb.SystemDrainResp) - drainReq := new(mgmtpb.PoolDrainReq) - drainReq.Sys = req.Sys - ranksMap := make(map[ranklist.Rank]struct{}) for _, r := range hitRanks.Ranks() { ranksMap[r] = struct{}{} } poolRanks := make(map[string]*ranklist.RankSet) - poolIDs := []string{} + poolIDs := []string{} // Label or UUID. for _, ps := range psList { currentRanks := ps.Storage.CurrentRanks() - poolID := ps.PoolUUID.String() + + // Label preferred over UUID. + poolID := ps.PoolLabel + if poolID == "" { + poolID = ps.PoolUUID.String() + } svc.log.Tracef("pool-service detected: id %s, ranks %v", poolID, currentRanks) @@ -1136,11 +1137,17 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) sort.Strings(poolIDs) + resp := new(mgmtpb.SystemDrainResp) + drainReq := new(mgmtpb.PoolDrainReq) + drainReq.Sys = req.Sys + for _, id := range poolIDs { rs := poolRanks[id] if rs.Count() == 0 { continue } + drained := ranklist.MustCreateRankSet("") + failed := make(map[string]*ranklist.RankSet) // Use our incoming request and just replace relevant parameters on each iteration. drainReq.Id = id @@ -1168,11 +1175,39 @@ func (svc *mgmtSvc) SystemDrain(ctx context.Context, req *mgmtpb.SystemDrainReq) svc.log.Tracef("pool-drain triggered from system-drain: %+v (req: %+v)", drainResp, drainReq) + // Each rank-drain failure message will produce a single result. + if drainResp.Status != 0 { + if _, exists := failed[errMsg]; !exists { + failed[errMsg] = ranklist.MustCreateRankSet("") + } + failed[errMsg].Add(r) + } else { + drained.Add(ranklist.Rank(drainReq.Rank)) + } + } + + // Single result generated for all ranks drained successfully. + if drained.Count() > 0 { + resp.Results = append(resp.Results, &mgmtpb.SystemDrainResp_DrainResult{ + PoolId: id, + Ranks: drained.String(), + }) + } + + var msgs []string + for msg := range failed { + msgs = append(msgs, msg) + } + sort.Strings(msgs) + + // Result generated for each failure message rank-group. + for _, msg := range msgs { resp.Results = append(resp.Results, &mgmtpb.SystemDrainResp_DrainResult{ - Status: drainResp.Status, - Msg: errMsg, - PoolId: drainReq.Id, - Ranks: fmt.Sprintf("%d", drainReq.Rank), + // Status already included in error message. + Status: -1, + Msg: msg, + PoolId: id, + Ranks: failed[msg].String(), }) } } diff --git a/src/control/server/mgmt_system_test.go b/src/control/server/mgmt_system_test.go index 67888cba981..50b58a5ab87 100644 --- a/src/control/server/mgmt_system_test.go +++ b/src/control/server/mgmt_system_test.go @@ -1847,6 +1847,7 @@ func TestServer_MgmtSvc_SystemDrain(t *testing.T) { drpcResp *mgmtpb.PoolDrainResp drpcErr error poolRanks map[string]string + useLabels bool expResp *mgmtpb.SystemDrainResp expErr error }{ @@ -1906,8 +1907,7 @@ func TestServer_MgmtSvc_SystemDrain(t *testing.T) { }, expResp: &mgmtpb.SystemDrainResp{ Results: []*mgmtpb.SystemDrainResp_DrainResult{ - {PoolId: test.MockUUID(1), Ranks: "0"}, - {PoolId: test.MockUUID(1), Ranks: "1"}, + {PoolId: test.MockUUID(1), Ranks: "0-1"}, {PoolId: test.MockUUID(2), Ranks: "1"}, }, }, @@ -1929,13 +1929,31 @@ func TestServer_MgmtSvc_SystemDrain(t *testing.T) { }, expResp: &mgmtpb.SystemDrainResp{ Results: []*mgmtpb.SystemDrainResp_DrainResult{ - {PoolId: test.MockUUID(1), Ranks: "0"}, - {PoolId: test.MockUUID(1), Ranks: "1"}, - {PoolId: test.MockUUID(1), Ranks: "2"}, - {PoolId: test.MockUUID(1), Ranks: "3"}, - {PoolId: test.MockUUID(2), Ranks: "1"}, - {PoolId: test.MockUUID(2), Ranks: "2"}, - {PoolId: test.MockUUID(2), Ranks: "3"}, + {PoolId: test.MockUUID(1), Ranks: "0-3"}, + {PoolId: test.MockUUID(2), Ranks: "1-3"}, + }, + }, + }, + "matching hosts; multiple pools; pool labels": { + req: &mgmtpb.SystemDrainReq{ + // Resolves to ranks 0-3. + Hosts: fmt.Sprintf("%s,%s", test.MockHostAddr(1), + test.MockHostAddr(2)), + }, + poolRanks: map[string]string{ + test.MockUUID(1): "0-4", + test.MockUUID(2): "1-7", + }, + useLabels: true, + drpcResp: &mgmtpb.PoolDrainResp{}, + expDrpcReqs: []*mgmtpb.PoolDrainReq{ + dReq(1, 0), dReq(1, 1), dReq(1, 2), dReq(1, 3), + dReq(2, 1), dReq(2, 2), dReq(2, 3), + }, + expResp: &mgmtpb.SystemDrainResp{ + Results: []*mgmtpb.SystemDrainResp_DrainResult{ + {PoolId: "00000001", Ranks: "0-3"}, + {PoolId: "00000002", Ranks: "1-3"}, }, }, }, @@ -1944,23 +1962,26 @@ func TestServer_MgmtSvc_SystemDrain(t *testing.T) { mockMember(t, 2, 0, "errored"), mockMember(t, 1, 0, "excluded"), }, - req: &mgmtpb.SystemDrainReq{Ranks: "1-2"}, - poolRanks: map[string]string{test.MockUUID(1): "1-2"}, - drpcResp: &mgmtpb.PoolDrainResp{Status: -1}, + req: &mgmtpb.SystemDrainReq{Ranks: "1-2"}, + poolRanks: map[string]string{ + test.MockUUID(1): "0-4", + test.MockUUID(2): "1-7", + }, + drpcResp: &mgmtpb.PoolDrainResp{Status: -1}, expDrpcReqs: []*mgmtpb.PoolDrainReq{ - dReq(1, 1), dReq(1, 2), + dReq(1, 1), dReq(1, 2), dReq(2, 1), dReq(2, 2), }, expResp: &mgmtpb.SystemDrainResp{ Results: []*mgmtpb.SystemDrainResp_DrainResult{ { PoolId: test.MockUUID(1), - Ranks: "1", + Ranks: "1-2", Status: -1, Msg: "DER_UNKNOWN(-1): Unknown error code -1", }, { - PoolId: test.MockUUID(1), - Ranks: "2", + PoolId: test.MockUUID(2), + Ranks: "1-2", Status: -1, Msg: "DER_UNKNOWN(-1): Unknown error code -1", }, @@ -1987,9 +2008,14 @@ func TestServer_MgmtSvc_SystemDrain(t *testing.T) { svc := mgmtSystemTestSetup(t, log, tc.members, nil) for uuidStr, ranksStr := range tc.poolRanks { + var label string + if tc.useLabels { + label = uuidStr[:8] + } addTestPoolService(t, svc.sysdb, &system.PoolService{ - PoolUUID: uuid.MustParse(uuidStr), - State: system.PoolServiceStateReady, + PoolUUID: uuid.MustParse(uuidStr), + PoolLabel: label, + State: system.PoolServiceStateReady, Storage: &system.PoolServiceStorage{ CurrentRankStr: ranksStr, }, diff --git a/src/proto/mgmt/system.proto b/src/proto/mgmt/system.proto index 4ee1e7cbf65..86cb26a02ef 100644 --- a/src/proto/mgmt/system.proto +++ b/src/proto/mgmt/system.proto @@ -93,10 +93,10 @@ message SystemDrainResp { int32 status = 1; // Status of the evict on the specific pool string msg = 2; // Error message if status indicates an error - string pool_id = 3; // uuid of pool - string ranks = 4; // rankset to have been drained on pool + string pool_id = 3; // Label or uuid of pool + string ranks = 4; // Rank-set that has encountered this result } - repeated DrainResult results = 1; // Results for individual pool-ranks drain calls. + repeated DrainResult results = 1; // Results for pool-ranks drain calls. } // SystemQueryReq supplies system query parameters. From ba2ebd40e6d84f143cd2188876bf7d5bb32b9838 Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Tue, 26 Nov 2024 12:28:30 +0000 Subject: [PATCH 6/6] correct doc typo Features: control Required-githooks: true Signed-off-by: Tom Nabarro --- docs/admin/pool_operations.md | 2 +- src/control/lib/control/system.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/admin/pool_operations.md b/docs/admin/pool_operations.md index 65f57c0fb70..bd6c2f0f3b0 100644 --- a/docs/admin/pool_operations.md +++ b/docs/admin/pool_operations.md @@ -1268,7 +1268,7 @@ command can be used. The command takes either a host-set or rank-set: To drain a set of hosts from all pools (drains all ranks on selected hosts): ```Bash -$ dmg system drain --hosts foo-[001-100] +$ dmg system drain --rank-hosts foo-[001-100] ``` To drain a set of ranks from all pools: diff --git a/src/control/lib/control/system.go b/src/control/lib/control/system.go index d2d64c5733e..3579a606f31 100644 --- a/src/control/lib/control/system.go +++ b/src/control/lib/control/system.go @@ -777,15 +777,12 @@ func LeaderQuery(ctx context.Context, rpcClient UnaryInvoker, req *LeaderQueryRe if err != nil { return nil, err } - rpcClient.Debugf("resp one: %+v", ur) resp := new(LeaderQueryResp) if err = convertMSResponse(ur, resp); err != nil { return nil, errors.Wrap(err, "converting MS to LeaderQuery resp") } - rpcClient.Debugf("resp one: %+v", resp) - req.SetHostList(resp.Replicas) ur, err = rpcClient.InvokeUnaryRPC(ctx, req) if err != nil {