diff --git a/src/control/cmd/daos/attribute.go b/src/control/cmd/daos/attribute.go index 9514dad70f0..a6e4662e524 100644 --- a/src/control/cmd/daos/attribute.go +++ b/src/control/cmd/daos/attribute.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2018-2021 Intel Corporation. +// (C) Copyright 2018-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,13 +7,10 @@ package main import ( - "fmt" - "io" "unsafe" + "github.com/daos-stack/daos/src/control/lib/daos" "github.com/pkg/errors" - - "github.com/daos-stack/daos/src/control/lib/txtfmt" ) /* @@ -21,61 +18,6 @@ import ( */ import "C" -type ( - attribute struct { - Name string `json:"name"` - Value []byte `json:"value,omitempty"` - } - - attrList []*attribute -) - -func (al attrList) asMap() map[string][]byte { - m := make(map[string][]byte) - for _, a := range al { - m[a.Name] = a.Value - } - return m -} - -func (al attrList) asList() []string { - names := make([]string, len(al)) - for i, a := range al { - names[i] = a.Name - } - return names -} - -func printAttributes(out io.Writer, header string, attrs ...*attribute) { - fmt.Fprintf(out, "%s\n", header) - - if len(attrs) == 0 { - fmt.Fprintln(out, " No attributes found.") - return - } - - nameTitle := "Name" - valueTitle := "Value" - titles := []string{nameTitle} - - table := []txtfmt.TableRow{} - for _, attr := range attrs { - row := txtfmt.TableRow{} - row[nameTitle] = attr.Name - if len(attr.Value) != 0 { - row[valueTitle] = string(attr.Value) - if len(titles) == 1 { - titles = append(titles, valueTitle) - } - } - table = append(table, row) - } - - tf := txtfmt.NewTableFormatter(titles...) - tf.InitWriter(out) - tf.Format(table) -} - type attrType int const ( @@ -83,13 +25,11 @@ const ( contAttr ) -func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrList, error) { +func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (daos.AttributeList, error) { var rc C.int expectedSize, totalSize := C.size_t(0), C.size_t(0) switch at { - case poolAttr: - rc = C.daos_pool_list_attr(hdl, nil, &totalSize, nil) case contAttr: rc = C.daos_cont_list_attr(hdl, nil, &totalSize, nil) default: @@ -109,8 +49,6 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis defer C.free(buf) switch at { - case poolAttr: - rc = C.daos_pool_list_attr(hdl, (*C.char)(buf), &totalSize, nil) case contAttr: rc = C.daos_cont_list_attr(hdl, (*C.char)(buf), &totalSize, nil) default: @@ -130,9 +68,9 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis return getDaosAttributes(hdl, at, attrNames) } - attrs := make([]*attribute, len(attrNames)) + attrs := make(daos.AttributeList, len(attrNames)) for i, name := range attrNames { - attrs[i] = &attribute{Name: name} + attrs[i] = &daos.Attribute{Name: name} } return attrs, nil @@ -141,7 +79,7 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis // getDaosAttributes fetches the values for the given list of attribute names. // Uses the bulk attribute fetch API to minimize roundtrips. -func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrList, error) { +func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (daos.AttributeList, error) { if len(names) == 0 { attrList, err := listDaosAttributes(hdl, at, false) if err != nil { @@ -171,8 +109,6 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi attrSizes := make([]C.size_t, numAttr) var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil) case contAttr: rc = C.daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil) default: @@ -199,8 +135,6 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // Do the actual fetch of all values in one go. switch at { - case poolAttr: - rc = C.daos_pool_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil) case contAttr: rc = C.daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil) default: @@ -214,9 +148,9 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // Note that we are copying the values into Go-managed byte slices // for safety and simplicity so that we can free the C memory as soon // as this function exits. - attrs := make([]*attribute, numAttr) + attrs := make(daos.AttributeList, numAttr) for i, name := range names { - attrs[i] = &attribute{ + attrs[i] = &daos.Attribute{ Name: name, Value: C.GoBytes(attrValues[i], C.int(attrSizes[i])), } @@ -228,7 +162,7 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // getDaosAttribute fetches the value for the given attribute name. // NB: For operations involving multiple attributes, the getDaosAttributes() // function is preferred for efficiency. -func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*attribute, error) { +func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*daos.Attribute, error) { attrs, err := getDaosAttributes(hdl, at, []string{name}) if err != nil { return nil, err @@ -241,7 +175,7 @@ func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*attribute // setDaosAttributes sets the values for the given list of attribute names. // Uses the bulk attribute set API to minimize roundtrips. -func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { +func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs daos.AttributeList) error { if len(attrs) == 0 { return nil } @@ -277,8 +211,6 @@ func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { attrCount := C.int(len(attrs)) var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil) case contAttr: rc = C.daos_cont_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil) default: @@ -291,12 +223,12 @@ func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { // setDaosAttribute sets the value for the given attribute name. // NB: For operations involving multiple attributes, the setDaosAttributes() // function is preferred for efficiency. -func setDaosAttribute(hdl C.daos_handle_t, at attrType, attr *attribute) error { +func setDaosAttribute(hdl C.daos_handle_t, at attrType, attr *daos.Attribute) error { if attr == nil { return errors.Errorf("nil %T", attr) } - return setDaosAttributes(hdl, at, attrList{attr}) + return setDaosAttributes(hdl, at, daos.AttributeList{attr}) } func delDaosAttribute(hdl C.daos_handle_t, at attrType, name string) error { @@ -305,8 +237,6 @@ func delDaosAttribute(hdl C.daos_handle_t, at attrType, name string) error { var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_del_attr(hdl, 1, &attrName, nil) case contAttr: rc = C.daos_cont_del_attr(hdl, 1, &attrName, nil) default: diff --git a/src/control/cmd/daos/container.go b/src/control/cmd/daos/container.go index e69c78a370e..c33335cd524 100644 --- a/src/control/cmd/daos/container.go +++ b/src/control/cmd/daos/container.go @@ -19,6 +19,7 @@ import ( "github.com/jessevdk/go-flags" "github.com/pkg/errors" + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/txtfmt" "github.com/daos-stack/daos/src/control/lib/ui" @@ -221,14 +222,14 @@ func queryContainer(poolUUID, contUUID uuid.UUID, poolHandle, contHandle C.daos_ return ci, nil } -func (cmd *containerBaseCmd) connectPool(flags C.uint, ap *C.struct_cmd_args_s) (func(), error) { +func (cmd *containerBaseCmd) connectPool(flags daos.PoolConnectFlag, ap *C.struct_cmd_args_s) (func(), error) { if err := cmd.poolBaseCmd.connectPool(flags); err != nil { return nil, err } if ap != nil { ap.pool = cmd.cPoolHandle - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { cmd.disconnectPool() return nil, err } @@ -299,7 +300,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { cmd.poolBaseCmd.Args.Pool.UUID = pu } - disconnectPool, err := cmd.connectPool(C.DAOS_PC_RW, ap) + disconnectPool, err := cmd.connectPool(daos.PoolConnectFlagReadWrite, ap) if err != nil { return err } @@ -317,7 +318,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { defer cmd.closeContainer() var ci *daos.ContainerInfo - ci, err = queryContainer(cmd.poolUUID, cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) + ci, err = queryContainer(cmd.pool.UUID(), cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) if err != nil { if errors.Cause(err) != daos.NoPermission { return errors.Wrapf(err, "failed to query new container %s", contID) @@ -327,7 +328,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { cmd.Errorf("container %s was created, but query failed", contID) ci = new(daos.ContainerInfo) - ci.PoolUUID = cmd.poolUUID + ci.PoolUUID = cmd.pool.UUID() ci.Type = cmd.Type.String() ci.ContainerUUID = cmd.contUUID ci.ContainerLabel = cmd.Args.Label @@ -406,9 +407,9 @@ func (cmd *containerCreateCmd) contCreate() (string, error) { } if len(cmd.Attrs.ParsedProps) != 0 { - attrs := make(attrList, 0, len(cmd.Attrs.ParsedProps)) + attrs := make(daos.AttributeList, 0, len(cmd.Attrs.ParsedProps)) for key, val := range cmd.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ + attrs = append(attrs, &daos.Attribute{ Name: key, Value: []byte(val), }) @@ -575,7 +576,7 @@ func (cmd *existingContainerCmd) resolveContainerPath(ap *C.struct_cmd_args_s) ( if err != nil { return } - cmd.poolBaseCmd.poolUUID = cmd.poolBaseCmd.Args.Pool.UUID + //cmd.poolBaseCmd.poolUUID = cmd.poolBaseCmd.Args.Pool.UUID cmd.poolBaseCmd.Args.Pool.Label = C.GoString(&ap.pool_str[0]) cmd.Args.Container.UUID, err = uuidFromC(ap.c_uuid) @@ -627,7 +628,7 @@ func (cmd *existingContainerCmd) resolveAndConnect(contFlags C.uint, ap *C.struc } var cleanupPool func() - cleanupPool, err = cmd.connectPool(C.DAOS_PC_RO, ap) + cleanupPool, err = cmd.connectPool(daos.PoolConnectFlagReadOnly, ap) if err != nil { return } @@ -651,7 +652,7 @@ func (cmd *existingContainerCmd) resolveAndConnect(contFlags C.uint, ap *C.struc }, nil } -func (cmd *existingContainerCmd) getAttr(name string) (*attribute, error) { +func (cmd *existingContainerCmd) getAttr(name string) (*daos.Attribute, error) { return getDaosAttribute(cmd.cContHandle, contAttr, name) } @@ -731,7 +732,7 @@ func printContainers(out io.Writer, contIDs []*ContainerID) { } func (cmd *containerListCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } @@ -990,7 +991,7 @@ func (cmd *containerQueryCmd) Execute(_ []string) error { } defer cleanup() - ci, err := queryContainer(cmd.poolUUID, cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) + ci, err := queryContainer(cmd.pool.UUID(), cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) if err != nil { return errors.Wrapf(err, "failed to query container %s", @@ -1123,14 +1124,14 @@ func (cmd *containerListAttrsCmd) Execute(args []string) error { if cmd.JSONOutputEnabled() { if cmd.Verbose { - return cmd.OutputJSON(attrs.asMap(), nil) + return cmd.OutputJSON(attrs.AsMap(), nil) } - return cmd.OutputJSON(attrs.asList(), nil) + return cmd.OutputJSON(attrs.AsList(), nil) } var bld strings.Builder title := fmt.Sprintf("Attributes for container %s:", cmd.ContainerID()) - printAttributes(&bld, title, attrs...) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) @@ -1208,7 +1209,7 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error { } defer cleanup() - var attrs attrList + var attrs daos.AttributeList if len(cmd.Args.Attrs.ParsedProps) == 0 { attrs, err = listDaosAttributes(cmd.cContHandle, contAttr, true) } else { @@ -1229,7 +1230,7 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error { var bld strings.Builder title := fmt.Sprintf("Attributes for container %s:", cmd.ContainerID()) - printAttributes(&bld, title, attrs...) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) @@ -1278,9 +1279,9 @@ func (cmd *containerSetAttrCmd) Execute(args []string) error { } defer cleanup() - attrs := make(attrList, 0, len(cmd.Args.Attrs.ParsedProps)) + attrs := make(daos.AttributeList, 0, len(cmd.Args.Attrs.ParsedProps)) for key, val := range cmd.Args.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ + attrs = append(attrs, &daos.Attribute{ Name: key, Value: []byte(val), }) @@ -1472,7 +1473,7 @@ func (f *ContainerID) Complete(match string) (comps []flags.Completion) { } defer fini() - cleanup, err := pf.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := pf.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return } diff --git a/src/control/cmd/daos/health.go b/src/control/cmd/daos/health.go index 61f1d1df142..0fe6aeda7a5 100644 --- a/src/control/cmd/daos/health.go +++ b/src/control/cmd/daos/health.go @@ -9,6 +9,7 @@ package main import ( "fmt" "strings" + "unsafe" "github.com/google/uuid" @@ -16,11 +17,17 @@ import ( "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" "github.com/daos-stack/daos/src/control/logging" ) +/* +#include "util.h" +*/ +import "C" + type healthCmds struct { Check healthCheckCmd `command:"check" description:"Perform DAOS system health checks"` NetTest netTestCmd `command:"net-test" description:"Perform non-destructive DAOS networking tests"` @@ -62,6 +69,8 @@ func collectBuildInfo(log logging.Logger, shi *daos.SystemHealthInfo) error { } func (cmd *healthCheckCmd) Execute([]string) error { + ctx := cmd.MustLogCtx() + // TODO (DAOS-10028): Move this logic into the daos package once the API is available. systemHealth := &daos.SystemHealthInfo{ ComponentBuildInfo: make(map[string]daos.ComponentBuild), @@ -72,7 +81,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { return err } - sysInfo, err := cmd.apiProvider.GetSystemInfo(cmd.MustLogCtx()) + sysInfo, err := cmd.apiProvider.GetSystemInfo(ctx) if err != nil { cmd.Errorf("failed to query system information: %v", err) } @@ -80,7 +89,10 @@ func (cmd *healthCheckCmd) Execute([]string) error { cmd.Infof("Checking DAOS system: %s", systemHealth.SystemInfo.Name) - pools, err := getPoolList(cmd.Logger, cmd.SysName, true) + pools, err := api.GetPoolList(ctx, api.GetPoolListReq{ + SysName: cmd.SysName, + Query: true, + }) if err != nil { cmd.Errorf("failed to get pool list: %v", err) } @@ -88,13 +100,18 @@ func (cmd *healthCheckCmd) Execute([]string) error { for _, pool := range pools { systemHealth.Pools[pool.UUID] = pool - poolHdl, _, err := poolConnect(pool.UUID.String(), cmd.SysName, daos.PoolConnectFlagReadOnly, false) + pcResp, err := api.PoolConnect(ctx, api.PoolConnectReq{ + SysName: cmd.SysName, + ID: pool.UUID.String(), + Flags: daos.PoolConnectFlagReadOnly, + Query: false, + }) if err != nil { cmd.Errorf("failed to connect to pool %s: %v", pool.Label, err) continue } defer func() { - if err := poolDisconnectAPI(poolHdl); err != nil { + if err := pcResp.PoolConnection.Disconnect(ctx); err != nil { cmd.Errorf("failed to disconnect from pool %s: %v", pool.Label, err) } }() @@ -104,7 +121,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { if pool.DisabledTargets > 0 { queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) } - tpi, err := queryPool(poolHdl, queryMask) + tpi, err := pcResp.PoolConnection.Query(ctx, queryMask) if err != nil { cmd.Errorf("failed to query pool %s: %v", pool.Label, err) continue @@ -113,6 +130,13 @@ func (cmd *healthCheckCmd) Execute([]string) error { pool.DisabledRanks = tpi.DisabledRanks pool.DeadRanks = tpi.DeadRanks + /* temporary, until we get the container API bindings */ + var poolHdl C.daos_handle_t + if err := pcResp.PoolConnection.FillHandle(unsafe.Pointer(&poolHdl)); err != nil { + cmd.Errorf("failed to fill handle for pool %s: %v", pool.Label, err) + continue + } + poolConts, err := listContainers(poolHdl) if err != nil { cmd.Errorf("failed to list containers on pool %s: %v", pool.Label, err) diff --git a/src/control/cmd/daos/health_test.go b/src/control/cmd/daos/health_test.go index 95565b36bb2..14a0e4b9bda 100644 --- a/src/control/cmd/daos/health_test.go +++ b/src/control/cmd/daos/health_test.go @@ -11,54 +11,87 @@ import ( "testing" "github.com/dustin/go-humanize" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" - "github.com/daos-stack/daos/src/control/lib/ui" - "github.com/daos-stack/daos/src/control/logging" +) + +var ( + runSelfTestResult []*daos.SelfTestResult + runSelfTestErr error ) func RunSelfTest(ctx context.Context, cfg *daos.SelfTestConfig) ([]*daos.SelfTestResult, error) { - return []*daos.SelfTestResult{}, nil + return runSelfTestResult, runSelfTestErr } func TestDaos_netTestCmdExecute(t *testing.T) { - // Quickie smoke test for the UI -- will flesh out later. - var opts cliOptions - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - args := []string{ - "health", "net-test", - "--ranks", "0-3", - "--tags", "4-9", - "--size", "20 MiB", - "--rep-count", "2222", - "--bytes", "--verbose", - } - expArgs := netTestCmd{} - expArgs.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) - expArgs.Tags.Replace(ranklist.MustCreateRankSet("4-9")) - expArgs.XferSize.Bytes = 20 * humanize.MiByte - expArgs.RepCount = 2222 - expArgs.Verbose = true - expArgs.TpsBytes = true + baseArgs := test.JoinArgs(nil, "health", "net-test") - if err := parseOpts(args, &opts, log); err != nil { - t.Fatal(err) - } - cmpOpts := cmp.Options{ - cmpopts.IgnoreUnexported(netTestCmd{}), - cmp.Comparer(func(a, b ranklist.RankSet) bool { - return a.String() == b.String() - }), - cmp.Comparer(func(a, b ui.ByteSizeFlag) bool { - return a.String() == b.String() - }), - cmpopts.IgnoreTypes(cmdutil.LogCmd{}, cmdutil.JSONOutputCmd{}), + for name, tc := range map[string]struct { + args []string + expArgs netTestCmd + expErr error + }{ + "all set (long)": { + args: test.JoinArgs(baseArgs, + "--ranks", "0-3", + "--tags", "4-9", + "--size", "20 MiB", + "--rep-count", "2222", + "--max-inflight", "1234", + "--bytes", "--verbose", + ), + expArgs: func() netTestCmd { + cmd := netTestCmd{} + cmd.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) + cmd.Tags.Replace(ranklist.MustCreateRankSet("4-9")) + cmd.XferSize.Bytes = 20 * humanize.MiByte + cmd.RepCount = 2222 + cmd.MaxInflight = 1234 + cmd.Verbose = true + cmd.TpsBytes = true + return cmd + }(), + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, + "-r", "0-3", + "-t", "4-9", + "-s", "20 MiB", + "-c", "2222", + "-m", "1234", + "-y", "-v", + ), + expArgs: func() netTestCmd { + cmd := netTestCmd{} + cmd.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) + cmd.Tags.Replace(ranklist.MustCreateRankSet("4-9")) + cmd.XferSize.Bytes = 20 * humanize.MiByte + cmd.RepCount = 2222 + cmd.MaxInflight = 1234 + cmd.Verbose = true + cmd.TpsBytes = true + return cmd + }(), + }, + "selftest fails": { + args: []string{"health", "net-test"}, + expErr: errors.New("whoops"), + }, + } { + t.Run(name, func(t *testing.T) { + if tc.expErr != nil { + prevErr := runSelfTestErr + t.Cleanup(func() { + runSelfTestErr = prevErr + }) + runSelfTestErr = tc.expErr + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Health.NetTest") + }) } - test.CmpAny(t, "health net-test args", expArgs, opts.Health.NetTest, cmpOpts...) } diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 831a775db1a..34018515f8e 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -12,14 +12,13 @@ import ( "strings" "unsafe" - "github.com/google/uuid" "github.com/pkg/errors" "github.com/daos-stack/daos/src/control/cmd/daos/pretty" - "github.com/daos-stack/daos/src/control/common" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" + "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" - "github.com/daos-stack/daos/src/control/logging" ) /* @@ -67,8 +66,7 @@ type PoolID struct { type poolBaseCmd struct { daosCmd - poolUUID uuid.UUID - + pool *api.PoolHandle cPoolHandle C.daos_handle_t Args struct { @@ -76,129 +74,63 @@ type poolBaseCmd struct { } `positional-args:"yes"` } -func (cmd *poolBaseCmd) poolUUIDPtr() *C.uchar { - if cmd.poolUUID == uuid.Nil { - cmd.Errorf("poolUUIDPtr(): nil UUID") - return nil - } - return (*C.uchar)(unsafe.Pointer(&cmd.poolUUID[0])) -} - func (cmd *poolBaseCmd) PoolID() ui.LabelOrUUIDFlag { return cmd.Args.Pool.LabelOrUUIDFlag } -// poolConnect is a convenience wrapper around poolConnectAPI. -func poolConnect(poolID, sysName string, flags uint, query bool) (C.daos_handle_t, *C.daos_pool_info_t, error) { - var cSysName *C.char - if sysName != "" { - cSysName = C.CString(sysName) - defer freeString(cSysName) - } - - cPoolID := C.CString(poolID) - defer freeString(cPoolID) - - var hdl C.daos_handle_t - var infoPtr *C.daos_pool_info_t - if query { - infoPtr = &C.daos_pool_info_t{ - pi_bits: C.ulong(daos.DefaultPoolQueryMask), - } - } - - return hdl, infoPtr, poolConnectAPI(cPoolID, cSysName, C.uint(flags), &hdl, infoPtr) -} - -// poolConnectAPI is a lower-level wrapper around daos_pool_connect(). -func poolConnectAPI(poolID, sysName *C.char, flags C.uint, hdl *C.daos_handle_t, info *C.daos_pool_info_t) error { - return daosError(C.daos_pool_connect(poolID, sysName, flags, hdl, info, nil)) -} - -// poolDisconnectAPI is a convenience wrapper around daos_pool_disconnect(). -func poolDisconnectAPI(hdl C.daos_handle_t) error { - // Hack for NLT fault injection testing: If the rc - // is -DER_NOMEM, retry once in order to actually - // shut down and release resources. - rc := C.daos_pool_disconnect(hdl, nil) - if rc == -C.DER_NOMEM { - rc = C.daos_pool_disconnect(hdl, nil) - // DAOS-8866, daos_pool_disconnect() might have failed, but worked anyway. - if rc == -C.DER_NO_HDL { - rc = -C.DER_SUCCESS - } - } - - return daosError(rc) -} - -func (cmd *poolBaseCmd) connectPool(flags C.uint) error { - sysName := cmd.SysName - var cSysName *C.char - if sysName != "" { - cSysName := C.CString(sysName) - defer freeString(cSysName) +func (cmd *poolBaseCmd) connectPool(flags daos.PoolConnectFlag) error { + req := api.PoolConnectReq{ + SysName: cmd.SysName, + Flags: flags, + Query: true, } switch { case cmd.PoolID().HasLabel(): - var poolInfo C.daos_pool_info_t - cLabel := C.CString(cmd.PoolID().Label) - defer freeString(cLabel) - - cmd.Debugf("connecting to pool: %s", cmd.PoolID().Label) - if err := poolConnectAPI(cLabel, cSysName, flags, &cmd.cPoolHandle, &poolInfo); err != nil { - return err - } - var err error - cmd.poolUUID, err = uuidFromC(poolInfo.pi_uuid) - if err != nil { - cmd.disconnectPool() - return err - } + req.ID = cmd.PoolID().Label case cmd.PoolID().HasUUID(): - cmd.poolUUID = cmd.PoolID().UUID - cmd.Debugf("connecting to pool: %s", cmd.poolUUID) - cUUIDstr := C.CString(cmd.poolUUID.String()) - defer freeString(cUUIDstr) - if err := poolConnectAPI(cUUIDstr, cSysName, flags, &cmd.cPoolHandle, nil); err != nil { - return err - } + req.ID = cmd.PoolID().UUID.String() default: return errors.New("no pool UUID or label supplied") } + resp, err := PoolConnect(cmd.MustLogCtx(), req) + if err != nil { + return err + } + cmd.pool = resp.PoolConnection + + if err := cmd.pool.FillHandle(unsafe.Pointer(&cmd.cPoolHandle)); err != nil { + if err := resp.PoolConnection.Disconnect(cmd.MustLogCtx()); err != nil { + cmd.Errorf("pool disconnect failed in cleanup: %v", err) + } + return err + } + return nil } func (cmd *poolBaseCmd) disconnectPool() { - cmd.Debugf("disconnecting pool %s", cmd.PoolID()) - if err := poolDisconnectAPI(cmd.cPoolHandle); err != nil { + if err := cmd.pool.Disconnect(cmd.MustLogCtx()); err != nil { cmd.Errorf("pool disconnect failed: %v", err) } } -func (cmd *poolBaseCmd) resolveAndConnect(flags C.uint, ap *C.struct_cmd_args_s) (func(), error) { +func (cmd *poolBaseCmd) resolveAndConnect(flags daos.PoolConnectFlag, ap *C.struct_cmd_args_s) (func(), error) { if err := cmd.connectPool(flags); err != nil { return nil, errors.Wrapf(err, "failed to connect to pool %s", cmd.PoolID()) } if ap != nil { - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { return nil, err } ap.pool = cmd.cPoolHandle - switch { - case cmd.PoolID().HasLabel(): - pLabel := C.CString(cmd.PoolID().Label) - defer freeString(pLabel) - C.strncpy(&ap.pool_str[0], pLabel, C.DAOS_PROP_LABEL_MAX_LEN) - case cmd.PoolID().HasUUID(): - pUUIDstr := C.CString(cmd.poolUUID.String()) - defer freeString(pUUIDstr) - C.strncpy(&ap.pool_str[0], pUUIDstr, C.DAOS_PROP_LABEL_MAX_LEN) - } + + pLabel := C.CString(cmd.pool.Label) + defer freeString(pLabel) + C.strncpy(&ap.pool_str[0], pLabel, C.DAOS_PROP_LABEL_MAX_LEN) } return func() { @@ -206,7 +138,7 @@ func (cmd *poolBaseCmd) resolveAndConnect(flags C.uint, ap *C.struct_cmd_args_s) }, nil } -func (cmd *poolBaseCmd) getAttr(name string) (*attribute, error) { +func (cmd *poolBaseCmd) getAttr(name string) (*daos.Attribute, error) { return getDaosAttribute(cmd.cPoolHandle, poolAttr, name) } @@ -216,7 +148,7 @@ type poolCmd struct { QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"query pool target info"` ListConts containerListCmd `command:"list-containers" alias:"list-cont" description:"list all containers in pool"` ListAttrs poolListAttrsCmd `command:"list-attr" alias:"list-attrs" alias:"lsattr" description:"list pool user-defined attributes"` - GetAttr poolGetAttrCmd `command:"get-attr" alias:"getattr" description:"get pool user-defined attribute"` + GetAttr poolListAttrCmd `command:"get-attr" alias:"getattr" description:"get pool user-defined attribute"` SetAttr poolSetAttrCmd `command:"set-attr" alias:"setattr" description:"set pool user-defined attribute"` DelAttr poolDelAttrCmd `command:"del-attr" alias:"delattr" description:"delete pool user-defined attribute"` AutoTest poolAutoTestCmd `command:"autotest" description:"verify setup with smoke tests"` @@ -228,130 +160,6 @@ type poolQueryCmd struct { HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } -func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { - if in == nil { - return nil - } - - return &daos.StorageUsageStats{ - Total: uint64(in.ps_space.s_total[mt]), - Free: uint64(in.ps_space.s_free[mt]), - Min: uint64(in.ps_free_min[mt]), - Max: uint64(in.ps_free_max[mt]), - Mean: uint64(in.ps_free_mean[mt]), - MediaType: daos.StorageMediaType(mt), - } -} - -func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *daos.PoolRebuildStatus { - if in == nil { - return nil - } - - out := &daos.PoolRebuildStatus{ - Status: int32(in.rs_errno), - } - if out.Status == 0 { - out.TotalObjects = uint64(in.rs_toberb_obj_nr) - out.Objects = uint64(in.rs_obj_nr) - out.Records = uint64(in.rs_rec_nr) - switch { - case in.rs_version == 0: - out.State = daos.PoolRebuildStateIdle - case C.get_rebuild_state(in) == C.DRS_COMPLETED: - out.State = daos.PoolRebuildStateDone - default: - out.State = daos.PoolRebuildStateBusy - } - } - - return out -} - -func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) { - poolInfo := new(daos.PoolInfo) - - poolInfo.QueryMask = daos.PoolQueryMask(pinfo.pi_bits) - poolInfo.UUID = uuid.Must(uuidFromC(pinfo.pi_uuid)) - poolInfo.TotalTargets = uint32(pinfo.pi_ntargets) - poolInfo.DisabledTargets = uint32(pinfo.pi_ndisabled) - poolInfo.ActiveTargets = uint32(pinfo.pi_space.ps_ntargets) - poolInfo.TotalEngines = uint32(pinfo.pi_nnodes) - poolInfo.ServiceLeader = uint32(pinfo.pi_leader) - poolInfo.Version = uint32(pinfo.pi_map_ver) - poolInfo.State = daos.PoolServiceStateReady - if poolInfo.DisabledTargets > 0 { - poolInfo.State = daos.PoolServiceStateDegraded - } - - poolInfo.Rebuild = convertPoolRebuildStatus(&pinfo.pi_rebuild_st) - if poolInfo.QueryMask.HasOption(daos.PoolQueryOptionSpace) { - poolInfo.TierStats = []*daos.StorageUsageStats{ - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM), - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME), - } - } - - return poolInfo, nil -} - -func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { - var enabledRanks *C.d_rank_list_t - var disabledRanks *C.d_rank_list_t - defer func() { - C.d_rank_list_free(enabledRanks) - C.d_rank_list_free(disabledRanks) - }() - - var rc C.int - cPoolInfo := C.daos_pool_info_t{ - pi_bits: C.uint64_t(queryMask), - } - if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) && queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { - enaQm := queryMask - enaQm.ClearOptions(daos.PoolQueryOptionDisabledEngines) - cPoolInfo.pi_bits = C.uint64_t(enaQm) - rc = C.daos_pool_query(poolHdl, &enabledRanks, &cPoolInfo, nil, nil) - if err := daosError(rc); err != nil { - return nil, err - } - - /* second query to just get disabled ranks */ - rc = C.daos_pool_query(poolHdl, &disabledRanks, nil, nil, nil) - } else if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) { - rc = C.daos_pool_query(poolHdl, &enabledRanks, &cPoolInfo, nil, nil) - } else if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { - rc = C.daos_pool_query(poolHdl, &disabledRanks, &cPoolInfo, nil, nil) - } else { - rc = C.daos_pool_query(poolHdl, nil, &cPoolInfo, nil, nil) - } - - if err := daosError(rc); err != nil { - return nil, err - } - - poolInfo, err := convertPoolInfo(&cPoolInfo) - if err != nil { - return nil, err - } - poolInfo.QueryMask = queryMask - - if enabledRanks != nil { - poolInfo.EnabledRanks, err = rankSetFromC(enabledRanks) - if err != nil { - return nil, err - } - } - if disabledRanks != nil { - poolInfo.DisabledRanks, err = rankSetFromC(disabledRanks) - if err != nil { - return nil, err - } - } - - return poolInfo, nil -} - func (cmd *poolQueryCmd) Execute(_ []string) error { queryMask := daos.DefaultPoolQueryMask if cmd.HealthOnly { @@ -360,15 +168,14 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { if cmd.ShowEnabledRanks { queryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) } - queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - poolInfo, err := queryPool(cmd.cPoolHandle, queryMask) + poolInfo, err := cmd.pool.Query(cmd.MustLogCtx(), queryMask) if err != nil { return errors.Wrapf(err, "failed to query pool %q", cmd.PoolID()) } @@ -391,73 +198,20 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { type poolQueryTargetsCmd struct { poolBaseCmd - Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"` - Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"` -} - -// For using the pretty printer that dmg uses for this target info. -func convertPoolTargetInfo(ptinfo *C.daos_target_info_t) (*daos.PoolQueryTargetInfo, error) { - pqti := new(daos.PoolQueryTargetInfo) - pqti.Type = daos.PoolQueryTargetType(ptinfo.ta_type) - pqti.State = daos.PoolQueryTargetState(ptinfo.ta_state) - pqti.Space = []*daos.StorageUsageStats{ - { - Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_SCM]), - Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_SCM]), - MediaType: C.DAOS_MEDIA_SCM, - }, - { - Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_NVME]), - Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_NVME]), - MediaType: C.DAOS_MEDIA_NVME, - }, - } - - return pqti, nil + Rank uint32 `long:"rank" required:"1" description:"Engine rank of the target(s) to be queried"` + Targets ui.RankSetFlag `long:"target-idx" description:"Comma-separated list of target index(es) to be queried (default: all)"` } func (cmd *poolQueryTargetsCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - var idxList []uint32 - if err = common.ParseNumberList(cmd.Targets, &idxList); err != nil { - return errors.WithMessage(err, "parsing target list") - } - - if len(idxList) == 0 { - pi, err := queryPool(cmd.cPoolHandle, daos.HealthOnlyPoolQueryMask) - if err != nil || (pi.TotalTargets == 0 || pi.TotalEngines == 0) { - if err != nil { - return errors.Wrap(err, "pool query failed") - } - return errors.New("failed to derive target count from pool query") - } - tgtCount := pi.TotalTargets / pi.TotalEngines - for i := uint32(0); i < tgtCount; i++ { - idxList = append(idxList, i) - } - } - - ptInfo := new(C.daos_target_info_t) - var rc C.int - - infos := make([]*daos.PoolQueryTargetInfo, 0, len(idxList)) - for tgt := 0; tgt < len(idxList); tgt++ { - rc = C.daos_pool_query_target(cmd.cPoolHandle, C.uint32_t(idxList[tgt]), C.uint32_t(cmd.Rank), ptInfo, nil) - if err := daosError(rc); err != nil { - return errors.Wrapf(err, - "failed to query pool %s rank:target %d:%d", cmd.poolUUID, cmd.Rank, idxList[tgt]) - } - - tgtInfo, err := convertPoolTargetInfo(ptInfo) - if err != nil { - return err - } - infos = append(infos, tgtInfo) + infos, err := cmd.pool.QueryTargets(cmd.MustLogCtx(), ranklist.Rank(cmd.Rank), &cmd.Targets.RankSet) + if err != nil { + return errors.Wrapf(err, "failed to query targets for pool %s", cmd.PoolID()) } if cmd.JSONOutputEnabled() { @@ -483,35 +237,44 @@ type poolListAttrsCmd struct { } func (cmd *poolListAttrsCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - attrs, err := listDaosAttributes(cmd.cPoolHandle, poolAttr, cmd.Verbose) - if err != nil { - return errors.Wrapf(err, - "failed to list attributes for pool %s", cmd.poolUUID) + var attrs daos.AttributeList + if !cmd.Verbose { + attrNames, err := cmd.pool.ListAttributes(cmd.MustLogCtx()) + if err != nil { + return errors.Wrapf(err, "failed to list attributes for pool %s", cmd.PoolID()) + } + attrs = attrListFromNames(attrNames) + } else { + var err error + attrs, err = cmd.pool.GetAttributes(cmd.MustLogCtx()) + if err != nil { + return errors.Wrapf(err, "failed to get attributes for pool %s", cmd.PoolID()) + } } if cmd.JSONOutputEnabled() { if cmd.Verbose { - return cmd.OutputJSON(attrs.asMap(), nil) + return cmd.OutputJSON(attrs.AsMap(), nil) } - return cmd.OutputJSON(attrs.asList(), nil) + return cmd.OutputJSON(attrs.AsList(), nil) } var bld strings.Builder - title := fmt.Sprintf("Attributes for pool %s:", cmd.poolUUID) - printAttributes(&bld, title, attrs...) + title := fmt.Sprintf("Attributes for pool %s:", cmd.PoolID()) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) return nil } -type poolGetAttrCmd struct { +type poolListAttrCmd struct { poolBaseCmd Args struct { @@ -519,19 +282,14 @@ type poolGetAttrCmd struct { } `positional-args:"yes"` } -func (cmd *poolGetAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) +func (cmd *poolListAttrCmd) Execute(_ []string) error { + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - var attrs attrList - if len(cmd.Args.Attrs.ParsedProps) == 0 { - attrs, err = listDaosAttributes(cmd.cPoolHandle, poolAttr, true) - } else { - attrs, err = getDaosAttributes(cmd.cPoolHandle, poolAttr, cmd.Args.Attrs.ParsedProps.ToSlice()) - } + attrs, err := cmd.pool.GetAttributes(cmd.MustLogCtx(), cmd.Args.Attrs.ParsedProps.ToSlice()...) if err != nil { return errors.Wrapf(err, "failed to get attributes for pool %s", cmd.PoolID()) } @@ -546,7 +304,7 @@ func (cmd *poolGetAttrCmd) Execute(_ []string) error { var bld strings.Builder title := fmt.Sprintf("Attributes for pool %s:", cmd.PoolID()) - printAttributes(&bld, title, attrs...) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) @@ -562,7 +320,7 @@ type poolSetAttrCmd struct { } func (cmd *poolSetAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } @@ -572,17 +330,18 @@ func (cmd *poolSetAttrCmd) Execute(_ []string) error { return errors.New("attribute name and value are required") } - attrs := make(attrList, 0, len(cmd.Args.Attrs.ParsedProps)) + attrs := make(daos.AttributeList, 0, len(cmd.Args.Attrs.ParsedProps)) for key, val := range cmd.Args.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ + attrs = append(attrs, &daos.Attribute{ Name: key, Value: []byte(val), }) } - if err := setDaosAttributes(cmd.cPoolHandle, poolAttr, attrs); err != nil { + if err := cmd.pool.SetAttributes(cmd.MustLogCtx(), attrs...); err != nil { return errors.Wrapf(err, "failed to set attributes on pool %s", cmd.PoolID()) } + cmd.Infof("Attributes successfully set on pool %q", cmd.PoolID()) return nil } @@ -591,22 +350,23 @@ type poolDelAttrCmd struct { poolBaseCmd Args struct { - Name string `positional-arg-name:"" required:"1"` + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]" required:"1"` } `positional-args:"yes"` } func (cmd *poolDelAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } defer cleanup() - if err := delDaosAttribute(cmd.cPoolHandle, poolAttr, cmd.Args.Name); err != nil { - return errors.Wrapf(err, - "failed to delete attribute %q on pool %s", - cmd.Args.Name, cmd.poolUUID) + attrNames := cmd.Args.Attrs.ParsedProps.ToSlice() + attrsString := strings.Join(attrNames, ",") + if err := cmd.pool.DeleteAttributes(cmd.MustLogCtx(), attrNames...); err != nil { + return errors.Wrapf(err, "failed to delete attributes %s on pool %s", attrsString, cmd.PoolID()) } + cmd.Infof("Attribute(s) %s successfully deleted on pool %q", attrsString, cmd.PoolID()) return nil } @@ -625,14 +385,14 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { } defer deallocCmdArgs() - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } defer cleanup() ap.pool = cmd.cPoolHandle - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { return err } ap.p_op = C.POOL_AUTOTEST @@ -649,108 +409,12 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { rc := C.pool_autotest_hdlr(ap) if err := daosError(rc); err != nil { - return errors.Wrapf(err, "failed to run autotest for pool %s", - cmd.poolUUID) + return errors.Wrapf(err, "failed to run autotest for pool %s", cmd.PoolID()) } return nil } -func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos.PoolInfo, error) { - var cSysName *C.char - if sysName != "" { - cSysName := C.CString(sysName) - defer freeString(cSysName) - } - - var cPools []C.daos_mgmt_pool_info_t - for { - var rc C.int - var poolCount C.size_t - - // First, fetch the total number of pools in the system. - // We may not have access to all of them, so this is an upper bound. - rc = C.daos_mgmt_list_pools(cSysName, &poolCount, nil, nil) - if err := daosError(rc); err != nil { - return nil, err - } - log.Debugf("pools in system: %d", poolCount) - - if poolCount < 1 { - return nil, nil - } - - // Now, we actually fetch the pools into the buffer that we've created. - cPools = make([]C.daos_mgmt_pool_info_t, poolCount) - rc = C.daos_mgmt_list_pools(cSysName, &poolCount, &cPools[0], nil) - err := daosError(rc) - if err == nil { - cPools = cPools[:poolCount] // adjust the slice to the number of pools retrieved - log.Debugf("fetched %d pools", len(cPools)) - break - } - if err == daos.StructTooSmall { - log.Notice("server-side pool list changed; re-fetching") - continue - } - log.Errorf("failed to fetch pool list: %s", err) - return nil, err - } - - pools := make([]*daos.PoolInfo, 0, len(cPools)) - for i := 0; i < len(cPools); i++ { - cPool := &cPools[i] - - svcRanks, err := rankSetFromC(cPool.mgpi_svc) - if err != nil { - return nil, err - } - poolUUID, err := uuidFromC(cPool.mgpi_uuid) - if err != nil { - return nil, err - } - poolLabel := C.GoString(cPool.mgpi_label) - - var pool *daos.PoolInfo - if queryEnabled { - poolHandle, poolInfo, err := poolConnect(poolUUID.String(), sysName, daos.PoolConnectFlagReadOnly, true) - if err != nil { - log.Errorf("failed to connect to pool %q: %s", poolLabel, err) - continue - } - - var qErr error - pool, qErr = convertPoolInfo(poolInfo) - if qErr != nil { - log.Errorf("failed to query pool %q: %s", poolLabel, qErr) - } - if err := poolDisconnectAPI(poolHandle); err != nil { - log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) - } - if qErr != nil { - continue - } - - // Add a few missing pieces that the query doesn't fill in. - pool.Label = poolLabel - pool.ServiceReplicas = svcRanks.Ranks() - } else { - // Just populate the basic info. - pool = &daos.PoolInfo{ - UUID: poolUUID, - Label: poolLabel, - ServiceReplicas: svcRanks.Ranks(), - State: daos.PoolServiceStateReady, - } - } - - pools = append(pools, pool) - } - - log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) - return pools, nil -} - type poolListCmd struct { daosCmd Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` @@ -758,7 +422,9 @@ type poolListCmd struct { } func (cmd *poolListCmd) Execute(_ []string) error { - pools, err := getPoolList(cmd.Logger, cmd.SysName, !cmd.NoQuery) + pools, err := GetPoolList(cmd.MustLogCtx(), api.GetPoolListReq{ + Query: !cmd.NoQuery, + }) if err != nil { return err } diff --git a/src/control/cmd/daos/pool_test.go b/src/control/cmd/daos/pool_test.go new file mode 100644 index 00000000000..2ccc02e61e8 --- /dev/null +++ b/src/control/cmd/daos/pool_test.go @@ -0,0 +1,475 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/dustin/go-humanize" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/lib/ui" +) + +var ( + defaultPoolInfo *daos.PoolInfo = &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), + Label: "test-pool", + TotalTargets: 48, + TotalEngines: 3, + ActiveTargets: 48, + DisabledTargets: 0, + Version: 1, + ServiceLeader: 2, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 64 * humanize.TByte, + Free: 16 * humanize.TByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 1 * humanize.PByte, + Free: 512 * humanize.TByte, + }, + }, + } +) + +var ( + defaultGetPoolListResult = []*daos.PoolInfo{ + defaultPoolInfo, + } + + getPoolListResult []*daos.PoolInfo = defaultGetPoolListResult + getPoolListErr error +) + +func GetPoolList(ctx context.Context, req api.GetPoolListReq) ([]*daos.PoolInfo, error) { + return getPoolListResult, getPoolListErr +} + +func TestDaos_poolListCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "list") + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolListCmd + }{ + "all set (long)": { + args: test.JoinArgs(baseArgs, "--verbose", "--no-query"), + expArgs: poolListCmd{ + NoQuery: true, + Verbose: true, + }, + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, "-v", "-n"), + expArgs: poolListCmd{ + NoQuery: true, + Verbose: true, + }, + }, + "query fails": { + args: []string{"pool", "list"}, + expErr: errors.New("whoops"), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.expErr != nil { + prevErr := getPoolListErr + t.Cleanup(func() { + getPoolListErr = prevErr + }) + getPoolListErr = tc.expErr + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.List") + }) + } +} + +var ( + defaultPoolConnectResp *api.PoolConnectResp = &api.PoolConnectResp{ + PoolConnection: &api.PoolHandle{}, + PoolInfo: defaultPoolInfo, + } + + poolConnectResp *api.PoolConnectResp = defaultPoolConnectResp + poolConnectErr error +) + +func PoolConnect(ctx context.Context, req api.PoolConnectReq) (*api.PoolConnectResp, error) { + return poolConnectResp, poolConnectErr +} + +func TestDaos_poolQueryCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "query", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolQueryCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "all set (long)": { + args: test.JoinArgs(baseArgs, "--show-enabled", "--health-only"), + expArgs: poolQueryCmd{ + ShowEnabledRanks: true, + HealthOnly: true, + }, + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, "-e", "-t"), + expArgs: poolQueryCmd{ + ShowEnabledRanks: true, + HealthOnly: true, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.Query") + }) + } +} + +func TestDaos_poolQueryTargetsCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "query-targets", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolQueryTargetsCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--rank=2", "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: test.JoinArgs(baseArgs[:len(baseArgs)-1], "--rank=2"), + expErr: errors.New("no pool UUID or label supplied"), + }, + "missing rank argument": { + args: baseArgs, + expErr: errors.New("required flag"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, "--rank=2"), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "success (rank only)": { + args: test.JoinArgs(baseArgs, "--rank=2"), + expArgs: poolQueryTargetsCmd{ + Rank: 2, + }, + }, + "success (rank and target)": { + args: test.JoinArgs(baseArgs, "--rank=2", "--target-idx=1,2"), + expArgs: poolQueryTargetsCmd{ + Rank: 2, + Targets: ui.RankSetFlag{ + RankSet: *ranklist.MustCreateRankSet("1,2"), + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.QueryTargets") + }) + } +} + +func TestDaos_poolSetAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "set-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + keyValArg := "key1:val1,key2:val2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolSetAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad", keyValArg), + expErr: errors.New("unknown flag"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, keyValArg), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "missing required arguments": { + args: baseArgs, + expErr: errors.New("required argument"), + }, + "malformed required arguments": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: errors.New("invalid property"), + }, + "success": { + args: test.JoinArgs(baseArgs, keyValArg), + expArgs: poolSetAttrCmd{ + Args: struct { + Attrs ui.SetPropertiesFlag `positional-arg-name:"key:val[,key:val...]" required:"1"` + }{ + Attrs: ui.SetPropertiesFlag{ + ParsedProps: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.SetAttr") + }) + } +} + +func TestDaos_poolGetAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "get-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolListAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "malformed arguments": { + args: test.JoinArgs(baseArgs, strings.ReplaceAll(keysOnlyArg, ",", ":")), + expErr: errors.New("key cannot contain"), + }, + "unknown key(s)": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: daos.Nonexistent, + }, + "success (one key)": { + args: test.JoinArgs(baseArgs, "one"), + expArgs: poolListAttrCmd{ + Args: struct { + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]"` + }{ + Attrs: ui.GetPropertiesFlag{ + ParsedProps: map[string]struct{}{ + "one": {}, + }, + }, + }, + }, + }, + "success (all keys)": { + args: baseArgs, + expArgs: poolListAttrCmd{}, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.GetAttr") + }) + } +} + +func TestDaos_poolDelAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "del-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolDelAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing required arguments": { + args: baseArgs, + expErr: errors.New("required argument"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "malformed arguments": { + args: test.JoinArgs(baseArgs, strings.ReplaceAll(keysOnlyArg, ",", ":")), + expErr: errors.New("key cannot contain"), + }, + "success (one key)": { + args: test.JoinArgs(baseArgs, "one"), + expArgs: poolDelAttrCmd{ + Args: struct { + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]" required:"1"` + }{ + Attrs: ui.GetPropertiesFlag{ + ParsedProps: map[string]struct{}{ + "one": {}, + }, + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.DelAttr") + }) + } +} + +func TestDaos_poolListAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "list-attr", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolListAttrsCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "success": { + args: baseArgs, + expArgs: poolListAttrsCmd{}, + }, + "success (verbose, short)": { + args: test.JoinArgs(baseArgs, "-V"), + expArgs: poolListAttrsCmd{ + Verbose: true, + }, + }, + "success (verbose, long)": { + args: test.JoinArgs(baseArgs, "--verbose"), + expArgs: poolListAttrsCmd{ + Verbose: true, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.ListAttrs") + }) + } +} diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index 75e7d9d13e9..51a5ddabbd3 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -338,3 +338,33 @@ func PrintPoolList(pools []*daos.PoolInfo, out io.Writer, verbose bool) error { return printPoolList(pools, out) } + +func PrintAttributes(out io.Writer, header string, attrs ...*daos.Attribute) { + fmt.Fprintf(out, "%s\n", header) + + if len(attrs) == 0 { + fmt.Fprintln(out, " No attributes found.") + return + } + + nameTitle := "Name" + valueTitle := "Value" + titles := []string{nameTitle} + + table := []txtfmt.TableRow{} + for _, attr := range attrs { + row := txtfmt.TableRow{} + row[nameTitle] = attr.Name + if len(attr.Value) != 0 { + row[valueTitle] = string(attr.Value) + if len(titles) == 1 { + titles = append(titles, valueTitle) + } + } + table = append(table, row) + } + + tf := txtfmt.NewTableFormatter(titles...) + tf.InitWriter(out) + tf.Format(table) +} diff --git a/src/control/cmd/daos/stubbed.go b/src/control/cmd/daos/stubbed.go index 4a08ad77255..11a7300701d 100644 --- a/src/control/cmd/daos/stubbed.go +++ b/src/control/cmd/daos/stubbed.go @@ -12,4 +12,6 @@ import "github.com/daos-stack/daos/src/control/lib/daos/api" var ( RunSelfTest = api.RunSelfTest + GetPoolList = api.GetPoolList + PoolConnect = api.PoolConnect ) diff --git a/src/control/cmd/daos/util.go b/src/control/cmd/daos/util.go index d5b128bf9a4..85c86bcd9af 100644 --- a/src/control/cmd/daos/util.go +++ b/src/control/cmd/daos/util.go @@ -336,3 +336,11 @@ func _writeDunsPath(path, ct string, poolUUID uuid.UUID, contUUID uuid.UUID) err return nil } + +func attrListFromNames(names []string) daos.AttributeList { + attrs := make(daos.AttributeList, len(names)) + for i, name := range names { + attrs[i] = &daos.Attribute{Name: name} + } + return attrs +} diff --git a/src/control/cmd/daos/util_test.go b/src/control/cmd/daos/util_test.go new file mode 100644 index 00000000000..575ed3fa7ec --- /dev/null +++ b/src/control/cmd/daos/util_test.go @@ -0,0 +1,59 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package main + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/daos-stack/daos/src/control/common/cmdutil" + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/lib/ui" + "github.com/daos-stack/daos/src/control/logging" +) + +func runCmdTest(t *testing.T, args []string, expCmd any, expErr error, cmdPath string, cmpOpts ...cmp.Option) { + t.Helper() + + var opts cliOptions + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + if err := parseOpts(args, &opts, log); err != nil { + test.CmpErr(t, expErr, err) + if expErr != nil { + return + } + } + + testCmd := reflect.ValueOf(opts) + for _, subCmd := range strings.Split(cmdPath, ".") { + testCmd = testCmd.FieldByName(subCmd) + if !testCmd.IsValid() || testCmd.IsZero() { + t.Fatalf("failed to select subcommand struct using %q", cmdPath) + } + } + + cmpOpts = append(cmpOpts, []cmp.Option{ + cmpopts.IgnoreUnexported(ui.GetPropertiesFlag{}, ui.SetPropertiesFlag{}, ui.PropertiesFlag{}), + cmpopts.IgnoreUnexported(testCmd.Interface()), + cmpopts.IgnoreTypes(cmdutil.LogCmd{}, cmdutil.JSONOutputCmd{}), + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + cmp.Comparer(func(a, b ui.ByteSizeFlag) bool { + return a.String() == b.String() + }), + }...) + test.CmpAny(t, fmt.Sprintf("%s args", cmdPath), expCmd, testCmd.Interface(), cmpOpts...) +} diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index 0b74128990b..782dae22459 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -698,8 +698,8 @@ func (cmd *poolQueryCmd) Execute(args []string) error { type poolQueryTargetsCmd struct { poolCmd - Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"` - Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"` + Rank uint32 `long:"rank" required:"1" description:"Engine rank of the target(s) to be queried"` + Targets ui.RankSetFlag `long:"target-idx" description:"Comma-separated list of target index(es) to be queried (default: all)"` } // Execute is run when PoolQueryTargetsCmd subcommand is activated @@ -707,11 +707,7 @@ func (cmd *poolQueryTargetsCmd) Execute(args []string) error { ctx := cmd.MustLogCtx() var tgtsList []uint32 - if len(cmd.Targets) > 0 { - if err := common.ParseNumberList(cmd.Targets, &tgtsList); err != nil { - return errors.WithMessage(err, "parsing target list") - } - } else { + if cmd.Targets.RankSet.Count() == 0 { pi, err := control.PoolQuery(ctx, cmd.ctlInvoker, &control.PoolQueryReq{ ID: cmd.PoolID().String(), QueryMask: daos.DefaultPoolQueryMask, @@ -726,6 +722,11 @@ func (cmd *poolQueryTargetsCmd) Execute(args []string) error { for i := uint32(0); i < tgtCount; i++ { tgtsList = append(tgtsList, i) } + } else { + tgtsList = make([]uint32, cmd.Targets.RankSet.Count()) + for i, rank := range cmd.Targets.RankSet.Ranks() { + tgtsList[i] = uint32(rank) + } } req := &control.PoolQueryTargetReq{ diff --git a/src/control/common/test/utils.go b/src/control/common/test/utils.go index ee685bf568a..d606c8d19e1 100644 --- a/src/control/common/test/utils.go +++ b/src/control/common/test/utils.go @@ -433,3 +433,11 @@ func MustLogContext(t *testing.T, log logging.Logger) context.Context { } return ctx } + +// JoinArgs creates a new string slice from a base string and optional +// additional string arguments. Does not modify the base string. +func JoinArgs(base []string, args ...string) []string { + joined := make([]string, len(base)) + copy(joined, base) + return append(joined, args...) +} diff --git a/src/control/lib/daos/api/api.go b/src/control/lib/daos/api/api.go index 51ef20e669a..e50c8697d5d 100644 --- a/src/control/lib/daos/api/api.go +++ b/src/control/lib/daos/api/api.go @@ -26,10 +26,6 @@ type ( } ) -func daosError(rc C.int) error { - return daos.ErrorFromRC(int(rc)) -} - func (api *api) isInitialized() bool { api.RLock() defer api.RUnlock() diff --git a/src/control/lib/daos/api/attribute.go b/src/control/lib/daos/api/attribute.go new file mode 100644 index 00000000000..81202275201 --- /dev/null +++ b/src/control/lib/daos/api/attribute.go @@ -0,0 +1,256 @@ +// +// (C) Copyright 2018-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "unsafe" + + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" +) + +/* +#include +#include + +#include +*/ +import "C" + +type attrType int + +const ( + poolAttr attrType = iota + contAttr +) + +func listDaosAttributes(hdl C.daos_handle_t, at attrType) ([]string, error) { + var rc C.int + expectedSize, totalSize := C.size_t(0), C.size_t(0) + + switch at { + case poolAttr: + rc = daos_pool_list_attr(hdl, nil, &totalSize, nil) + /*case contAttr: + rc = daos_cont_list_attr(hdl, nil, &totalSize, nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to list attributes") + } + + if totalSize < 1 { + return nil, nil + } + + attrNames := []string{} + expectedSize = totalSize + cNamesBuf := C.malloc(totalSize) + defer C.free(cNamesBuf) + + switch at { + case poolAttr: + rc = daos_pool_list_attr(hdl, (*C.char)(cNamesBuf), &totalSize, nil) + /*case contAttr: + rc = daos_cont_list_attr(hdl, (*C.char)(buf), &totalSize, nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to list attributes") + } + + if err := iterStringsBuf(cNamesBuf, expectedSize, func(name string) { + attrNames = append(attrNames, name) + }); err != nil { + return nil, err + } + + return attrNames, nil +} + +// getDaosAttributes fetches the values for the given list of attribute names. +// Uses the bulk attribute fetch API to minimize roundtrips. +func getDaosAttributes(hdl C.daos_handle_t, at attrType, reqAttrNames []string) (daos.AttributeList, error) { + if len(reqAttrNames) == 0 { + attrNameList, err := listDaosAttributes(hdl, at) + if err != nil { + return nil, errors.Wrap(err, "failed to list attributes") + } + reqAttrNames = attrNameList + } + numAttr := len(reqAttrNames) + + if numAttr == 0 { + return nil, nil + } + + // First, build a slice of C strings for the requested attribute names. + cAttrNames := make([]*C.char, numAttr) + for i, name := range reqAttrNames { + if name == "" { + return nil, errors.Wrapf(daos.InvalidInput, "empty attribute name at index %d", i) + } + cAttrNames[i] = C.CString(name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(cAttrNames) + + // Next, create a slice of C.size_t entries to hold the sizes of the values. + // We have to do this first in order to know the buffer sizes to allocate + // before fetching the actual values. + cAttrSizes := make([]C.size_t, numAttr) + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_get_attr(hdl, C.int(numAttr), &cAttrNames[0], nil, &cAttrSizes[0], nil) + /*case contAttr: + rc = daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to get attribute sizes") + } + + // Now, create a slice of buffers to hold the values. + cAttrValues := make([]unsafe.Pointer, numAttr) + defer func(valueSlice []unsafe.Pointer) { + for _, value := range valueSlice { + C.free(value) + } + }(cAttrValues) + for i, size := range cAttrSizes { + if size < 1 { + return nil, errors.Wrapf(daos.MiscError, "failed to get attribute %s: size is %d", reqAttrNames[i], size) + } + + cAttrValues[i] = C.malloc(size) + } + + // Do the actual fetch of all values in one go. + switch at { + case poolAttr: + rc = daos_pool_get_attr(hdl, C.int(numAttr), &cAttrNames[0], &cAttrValues[0], &cAttrSizes[0], nil) + /*case contAttr: + rc = daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to get attribute values") + } + + // Finally, create a slice of attribute structs to hold the results. + // Note that we are copying the values into Go-managed byte slices + // for safety and simplicity so that we can free the C memory as soon + // as this function exits. + attrs := make([]*daos.Attribute, numAttr) + for i, name := range reqAttrNames { + attrs[i] = &daos.Attribute{ + Name: name, + Value: C.GoBytes(cAttrValues[i], C.int(cAttrSizes[i])), + } + } + + return attrs, nil +} + +// setDaosAttributes sets the values for the given list of attribute names. +// Uses the bulk attribute set API to minimize roundtrips. +func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs daos.AttributeList) error { + if len(attrs) == 0 { + return errors.Wrap(daos.InvalidInput, "no attributes provided") + } + + // First, build a slice of C strings for the attribute names. + attrNames := make([]*C.char, len(attrs)) + for i, attr := range attrs { + if attr == nil { + return errors.Wrapf(daos.InvalidInput, "nil attribute at index %d", i) + } + if attr.Name == "" { + return errors.Wrapf(daos.InvalidInput, "empty attribute name at index %d", i) + } + attrNames[i] = C.CString(attr.Name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(attrNames) + + // Next, create a slice of C.size_t entries to hold the sizes of the values, + // and a slice of pointers to the actual values. + attrSizes := make([]C.size_t, len(attrs)) + attrValues := make([]unsafe.Pointer, len(attrs)) + for i, attr := range attrs { + attrSizes[i] = C.size_t(len(attr.Value)) + if attrSizes[i] == 0 { + return errors.Wrapf(daos.InvalidInput, "empty attribute value at index %d", i) + } + // NB: We are copying the values into C memory for safety and simplicity. + attrValues[i] = C.malloc(attrSizes[i]) + valSlice := unsafe.Slice((*byte)(attrValues[i]), attrSizes[i]) + copy(valSlice[:], attr.Value) + } + defer func(bufSlice []unsafe.Pointer) { + for _, buf := range bufSlice { + C.free(buf) + } + }(attrValues) + + attrCount := C.int(len(attrs)) + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_set_attr(hdl, attrCount, &attrNames[0], &attrValues[0], &attrSizes[0], nil) + /*case contAttr: + rc = daos_cont_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil)*/ + default: + return errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + + return errors.Wrap(daosError(rc), "failed to set attributes") +} + +// delDaosAttributes deletes the given attributes. +func delDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) error { + if len(names) == 0 { + return errors.Wrap(daos.InvalidInput, "no attribute names provided") + } + + attrNames := make([]*C.char, len(names)) + for i, name := range names { + if name == "" { + return errors.Wrapf(daos.InvalidInput, "empty attribute name at index %d", i) + } + attrNames[i] = C.CString(name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(attrNames) + + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_del_attr(hdl, C.int(len(attrNames)), &attrNames[0], nil) + /*case contAttr: + rc = daos_cont_del_attr(hdl, 1, &attrName, nil)*/ + default: + return errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + + return errors.Wrap(daosError(rc), "failed to delete attributes") +} diff --git a/src/control/lib/daos/api/errors.go b/src/control/lib/daos/api/errors.go index 6d1b4b665e3..fdb5193280e 100644 --- a/src/control/lib/daos/api/errors.go +++ b/src/control/lib/daos/api/errors.go @@ -6,8 +6,55 @@ package api -import "github.com/pkg/errors" +import ( + "context" + + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" +) + +/* +#include +*/ +import "C" var ( - ErrNoSystemRanks = errors.New("no ranks in system") + ErrNoSystemRanks = errors.New("no ranks in system") + ErrInvalidPoolHandle = errors.New("pool handle is nil or invalid") + + errInvalidContainerHandle = errors.New("container handle is nil or invalid") + errNilCtx = errors.New("nil context") + errNoCtxHdl = errors.New("no handle in context") ) + +// dfsError converts a return code from a DFS API +// call to a Go error. +func dfsError(rc C.int) error { + if rc == 0 { + return nil + } + + strErr := C.strerror(rc) + return errors.Errorf("DFS error %d: %s", rc, C.GoString(strErr)) +} + +// daosError converts a return code from a DAOS API +// call to a Go error. +func daosError(rc C.int) error { + return daos.ErrorFromRC(int(rc)) +} + +// ctxErr recasts a context error as a DAOS error. +func ctxErr(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, context.Canceled): + return errors.Wrap(daos.Canceled, "DAOS API context canceled") + case errors.Is(err, context.DeadlineExceeded): + return errors.Wrap(daos.TimedOut, "DAOS API context deadline exceeded") + default: + return errors.Wrap(daos.MiscError, "DAOS API context error") + } +} diff --git a/src/control/lib/daos/api/handle.go b/src/control/lib/daos/api/handle.go new file mode 100644 index 00000000000..a86555f786d --- /dev/null +++ b/src/control/lib/daos/api/handle.go @@ -0,0 +1,89 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "context" + "fmt" + "unsafe" + + "github.com/daos-stack/daos/src/control/logging" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +/* +#include + +#cgo LDFLAGS: -ldaos_common +*/ +import "C" + +type ( + // ctxHdlKey is a type used for storing handles as context values. + ctxHdlKey string + + // connHandle is an opaque type used to represent a DAOS connection (pool or container). + connHandle struct { + UUID uuid.UUID + Label string + daosHandle C.daos_handle_t + } +) + +// invalidate clears the handle so that it cannot be reused inadvertently. +func (ch *connHandle) invalidate() { + ch.UUID = uuid.Nil + ch.Label = "" + ch.daosHandle.cookie = 0 +} + +// FillHandle copies the handle to the supplied memory buffer. +// NB: Caller is responsible for keeping the copy in sync with +// this handle -- use of this method should be discouraged. +func (ch *connHandle) FillHandle(cHandle unsafe.Pointer) error { + if ch == nil || cHandle == nil { + return errors.New("invalid handle") + } + (*C.daos_handle_t)(cHandle).cookie = ch.daosHandle.cookie + + return nil +} + +// IsValid returns true if the pool or container handle is valid. +func (ch *connHandle) IsValid() bool { + return bool(daos_handle_is_valid(ch.daosHandle)) +} + +func (ch *connHandle) String() string { + return fmt.Sprintf("%s(%s):%t", ch.Label, logging.ShortUUID(ch.UUID), ch.IsValid()) +} + +// fromCtx retrieves the handle from the supplied context, if available. +func (ch *connHandle) fromCtx(ctx context.Context, key ctxHdlKey) error { + if ch == nil { + return errors.New("nil connHandle") + } + if ctx == nil { + return errNilCtx + } + hdl, ok := ctx.Value(key).(connHandle) + if !ok { + return errNoCtxHdl + } + + *ch = hdl + return nil +} + +// toCtx returns a new context with the handle stashed in it. +func (ch *connHandle) toCtx(ctx context.Context, key ctxHdlKey) context.Context { + if ch == nil { + return ctx + } + return context.WithValue(ctx, key, *ch) +} diff --git a/src/control/lib/daos/api/libdaos.go b/src/control/lib/daos/api/libdaos.go index d7c6bfed82d..74e79436816 100644 --- a/src/control/lib/daos/api/libdaos.go +++ b/src/control/lib/daos/api/libdaos.go @@ -12,10 +12,12 @@ package api #include #include #include +#include #cgo LDFLAGS: -lcart -lgurt -ldaos -ldaos_common */ import "C" +import "unsafe" func daos_init() C.int { return C.daos_init() @@ -29,6 +31,10 @@ func dc_agent_fini() { C.dc_agent_fini() } +func daos_handle_is_valid(handle C.daos_handle_t) C.bool { + return C.daos_handle_is_valid(handle) +} + func daos_mgmt_get_sys_info(sys *C.char, sys_info **C.struct_daos_sys_info) C.int { return C.daos_mgmt_get_sys_info(sys, sys_info) } @@ -36,3 +42,51 @@ func daos_mgmt_get_sys_info(sys *C.char, sys_info **C.struct_daos_sys_info) C.in func daos_mgmt_put_sys_info(sys_info *C.struct_daos_sys_info) { C.daos_mgmt_put_sys_info(sys_info) } + +func daos_pool_connect(poolID *C.char, sys *C.char, flags C.uint32_t, poolHdl *C.daos_handle_t, poolInfo *C.daos_pool_info_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_connect(poolID, sys, flags, poolHdl, poolInfo, ev) +} + +func daos_pool_disconnect(poolHdl C.daos_handle_t) C.int { + // Hack for NLT fault injection testing: If the rc + // is -DER_NOMEM, retry once in order to actually + // shut down and release resources. + rc := C.daos_pool_disconnect(poolHdl, nil) + if rc == -C.DER_NOMEM { + rc = C.daos_pool_disconnect(poolHdl, nil) + // DAOS-8866, daos_pool_disconnect() might have failed, but worked anyway. + if rc == -C.DER_NO_HDL { + rc = -C.DER_SUCCESS + } + } + + return rc +} + +func daos_pool_query(poolHdl C.daos_handle_t, rankList **C.d_rank_list_t, poolInfo *C.daos_pool_info_t, props *C.daos_prop_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_query(poolHdl, rankList, poolInfo, props, ev) +} + +func daos_pool_query_target(poolHdl C.daos_handle_t, tgt C.uint32_t, rank C.uint32_t, info *C.daos_target_info_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_query_target(poolHdl, tgt, rank, info, ev) +} + +func daos_pool_list_attr(poolHdl C.daos_handle_t, buf *C.char, size *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_list_attr(poolHdl, buf, size, ev) +} + +func daos_pool_get_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_get_attr(poolHdl, n, names, values, sizes, ev) +} + +func daos_pool_set_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_set_attr(poolHdl, n, names, values, sizes, ev) +} + +func daos_pool_del_attr(poolHdl C.daos_handle_t, n C.int, name **C.char, ev *C.struct_daos_event) C.int { + return C.daos_pool_del_attr(poolHdl, n, name, ev) +} + +func daos_mgmt_list_pools(sysName *C.char, poolCount *C.daos_size_t, pools *C.daos_mgmt_pool_info_t, ev *C.struct_daos_event) C.int { + return C.daos_mgmt_list_pools(sysName, poolCount, pools, ev) +} diff --git a/src/control/lib/daos/api/libdaos_stubs.go b/src/control/lib/daos/api/libdaos_stubs.go index 341b90bdd34..c4d86778d7a 100644 --- a/src/control/lib/daos/api/libdaos_stubs.go +++ b/src/control/lib/daos/api/libdaos_stubs.go @@ -11,13 +11,26 @@ package api import ( "unsafe" + "github.com/dustin/go-humanize" + "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" ) /* #include #include +#include + +#include "util.h" + +static inline void +set_rebuild_state(struct daos_rebuild_status *drs, int32_t state) +{ + drs->rs_state = state; +} */ import "C" @@ -33,6 +46,14 @@ func daos_fini() {} func dc_agent_fini() {} +var ( + daos_handle_is_valid_Bool C.bool = true +) + +func daos_handle_is_valid(handle C.daos_handle_t) C.bool { + return daos_handle_is_valid_Bool +} + var ( defaultSystemInfo *daos.SystemInfo = &daos.SystemInfo{ Name: build.DefaultSystemName, @@ -114,3 +135,546 @@ func daos_mgmt_put_sys_info(sys_info *C.struct_daos_sys_info) { C.free(unsafe.Pointer(sys_info.dsi_ms_ranks)) } } + +func daos_gds2cds(sus []*daos.StorageUsageStats) C.struct_daos_space { + return C.struct_daos_space{ + s_total: [2]C.uint64_t{ + C.uint64_t(sus[0].Total), + C.uint64_t(sus[1].Total), + }, + s_free: [2]C.uint64_t{ + C.uint64_t(sus[0].Free), + C.uint64_t(sus[1].Free), + }, + } +} + +func daos_gpi2cpi(gpi *daos.PoolInfo) *C.daos_pool_info_t { + cpi := &C.daos_pool_info_t{ + pi_uuid: uuidToC(gpi.UUID), + pi_ntargets: C.uint32_t(gpi.TotalTargets), + pi_nnodes: C.uint32_t(gpi.TotalEngines), + pi_ndisabled: C.uint32_t(gpi.DisabledTargets), + pi_map_ver: C.uint32_t(gpi.Version), + pi_leader: C.uint32_t(gpi.ServiceLeader), + pi_bits: C.uint64_t(gpi.QueryMask), + pi_rebuild_st: C.struct_daos_rebuild_status{ + rs_errno: C.int32_t(gpi.Rebuild.Status), + rs_obj_nr: C.uint64_t(gpi.Rebuild.Objects), + rs_rec_nr: C.uint64_t(gpi.Rebuild.Records), + }, + pi_space: C.struct_daos_pool_space{ + ps_ntargets: C.uint32_t(gpi.ActiveTargets), + ps_space: daos_gds2cds(gpi.TierStats), + ps_free_min: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Min), + C.uint64_t(gpi.TierStats[1].Min), + }, + ps_free_max: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Max), + C.uint64_t(gpi.TierStats[1].Max), + }, + ps_free_mean: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Mean), + C.uint64_t(gpi.TierStats[1].Mean), + }, + }, + } + + // some funky mismatch between the Go/C states... fix this later. + switch gpi.Rebuild.State { + case daos.PoolRebuildStateIdle: + cpi.pi_rebuild_st.rs_version = 0 + case daos.PoolRebuildStateBusy: + cpi.pi_rebuild_st.rs_version = 1 + C.set_rebuild_state(&cpi.pi_rebuild_st, C.DRS_IN_PROGRESS) + case daos.PoolRebuildStateDone: + cpi.pi_rebuild_st.rs_version = 1 + C.set_rebuild_state(&cpi.pi_rebuild_st, C.DRS_COMPLETED) + } + return cpi +} + +// defaultPoolInfo should be used to get a copy of the default pool info. +func defaultPoolInfo() *daos.PoolInfo { + return copyPoolInfo(&daos_default_PoolInfo) +} + +func copyPoolInfo(in *daos.PoolInfo) *daos.PoolInfo { + if in == nil { + return nil + } + + out := new(daos.PoolInfo) + *out = *in + + if in.Rebuild != nil { + out.Rebuild = new(daos.PoolRebuildStatus) + *out.Rebuild = *in.Rebuild + } + if in.TierStats != nil { + out.TierStats = make([]*daos.StorageUsageStats, len(in.TierStats)) + for i, s := range in.TierStats { + out.TierStats[i] = new(daos.StorageUsageStats) + *out.TierStats[i] = *s + } + } + if in.ServiceReplicas != nil { + out.ServiceReplicas = make([]ranklist.Rank, len(in.ServiceReplicas)) + copy(out.ServiceReplicas, in.ServiceReplicas) + } + if in.EnabledRanks != nil { + out.EnabledRanks = ranklist.NewRankSet() + out.EnabledRanks.Replace(in.EnabledRanks) + } + if in.DisabledRanks != nil { + out.DisabledRanks = ranklist.NewRankSet() + out.DisabledRanks.Replace(in.DisabledRanks) + } + + return out +} + +var ( + daos_default_PoolInfo daos.PoolInfo = daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: test.MockPoolUUID(42), + Label: "test-pool", + TotalTargets: 48, + TotalEngines: 3, + ActiveTargets: 32, + DisabledTargets: 16, + Version: 2, + ServiceLeader: 1, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + EnabledRanks: ranklist.MustCreateRankSet("0,2"), + DisabledRanks: ranklist.MustCreateRankSet("1"), + Rebuild: &daos.PoolRebuildStatus{ + Status: 0, + Objects: 1, + Records: 2, + State: daos.PoolRebuildStateBusy, + }, + TierStats: []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 64 * humanize.TByte, + Free: 16 * humanize.TByte, + Min: 1 * humanize.TByte, + Max: 4 * humanize.TByte, + Mean: 2 * humanize.TByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 64 * humanize.PByte, + Free: 16 * humanize.PByte, + Min: 1 * humanize.PByte, + Max: 4 * humanize.PByte, + Mean: 2 * humanize.PByte, + }, + }, + } + + daos_default_PoolQueryTargetInfo daos.PoolQueryTargetInfo = daos.PoolQueryTargetInfo{ + Type: daos.PoolQueryTargetType(1), + State: daos.PoolTargetStateUp, + Space: func() []*daos.StorageUsageStats { + tiStats := make([]*daos.StorageUsageStats, len(daos_default_PoolInfo.TierStats)) + for i, tier := range daos_default_PoolInfo.TierStats { + tiStats[i] = &daos.StorageUsageStats{ + MediaType: tier.MediaType, + Total: tier.Total, + Free: tier.Free, + } + } + return tiStats + }(), + } +) + +var ( + daos_default_pool_connect_Handle C.daos_handle_t = C.daos_handle_t{cookie: 42} + daos_default_pool_connect_Info *C.daos_pool_info_t = daos_gpi2cpi(&daos_default_PoolInfo) + daos_pool_connect_SetPoolID string + daos_pool_connect_SetSys string + daos_pool_connect_SetFlags daos.PoolConnectFlag + daos_pool_connect_QueryMask daos.PoolQueryMask + daos_pool_connect_Handle *C.daos_handle_t = &daos_default_pool_connect_Handle + daos_pool_connect_Info *C.daos_pool_info_t = daos_default_pool_connect_Info + daos_pool_connect_RC C.int = 0 +) + +func reset_daos_pool_connect() { + daos_pool_connect_SetPoolID = "" + daos_pool_connect_SetSys = "" + daos_pool_connect_SetFlags = 0 + daos_pool_connect_QueryMask = 0 + daos_pool_connect_Handle = &daos_default_pool_connect_Handle + daos_pool_connect_Info = daos_default_pool_connect_Info + daos_pool_connect_RC = 0 +} + +func daos_pool_connect(poolID *C.char, sys *C.char, flags C.uint32_t, poolHdl *C.daos_handle_t, poolInfo *C.daos_pool_info_t, ev *C.struct_daos_event) C.int { + if daos_pool_connect_RC != 0 { + return daos_pool_connect_RC + } + + // capture the parameters set by the test + daos_pool_connect_SetPoolID = C.GoString(poolID) + daos_pool_connect_SetSys = C.GoString(sys) + daos_pool_connect_SetFlags = daos.PoolConnectFlag(flags) + daos_pool_connect_QueryMask = daos.PoolQueryMask(poolInfo.pi_bits) + + // set the return values + poolHdl.cookie = daos_pool_connect_Handle.cookie + *poolInfo = *daos_pool_connect_Info + + return daos_pool_connect_RC +} + +var ( + daos_pool_disconnect_RC C.int = 0 +) + +func daos_pool_disconnect(poolHdl C.daos_handle_t) C.int { + return daos_pool_disconnect_RC +} + +var ( + daos_pool_query_PoolInfo *daos.PoolInfo = defaultPoolInfo() + daos_pool_query_RC C.int = 0 +) + +func reset_daos_pool_query() { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_RC = 0 +} + +func daos_pool_query(poolHdl C.daos_handle_t, rankList **C.d_rank_list_t, retPoolInfo *C.daos_pool_info_t, props *C.daos_prop_t, ev *C.struct_daos_event) C.int { + if daos_pool_query_RC != 0 { + return daos_pool_query_RC + } + + if retPoolInfo == nil { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.DisabledRanks) + return daos_pool_query_RC + } + + queryBits := retPoolInfo.pi_bits + *retPoolInfo = *daos_gpi2cpi(daos_pool_query_PoolInfo) + retPoolInfo.pi_bits = queryBits + + if queryBits&C.DPI_ENGINES_ENABLED != 0 { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.EnabledRanks) + } + if queryBits&C.DPI_ENGINES_DISABLED != 0 { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.DisabledRanks) + } + + if props != nil { + propEntries := unsafe.Slice(props.dpp_entries, props.dpp_nr) + for i := range propEntries { + switch propEntries[i].dpe_type { + case C.DAOS_PROP_PO_LABEL: + C.set_dpe_str(&propEntries[i], C.CString(daos_pool_query_PoolInfo.Label)) + case C.DAOS_PROP_PO_SVC_LIST: + rlPtr := ranklistFromGo(ranklist.RankSetFromRanks(daos_pool_query_PoolInfo.ServiceReplicas)) + C.set_dpe_val_ptr(&propEntries[i], (unsafe.Pointer)(rlPtr)) + } + } + } + + return daos_pool_query_RC +} + +var ( + daos_pool_query_target_SetTgt C.uint32_t = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_SetRank C.uint32_t = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_Info *daos.PoolQueryTargetInfo = &daos_default_PoolQueryTargetInfo + daos_pool_query_target_RC C.int = 0 +) + +func reset_daos_pool_query_target() { + daos_pool_query_target_SetTgt = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_SetRank = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_Info = &daos_default_PoolQueryTargetInfo + daos_pool_query_target_RC = 0 +} + +func daos_pool_query_target(poolHdl C.daos_handle_t, tgt C.uint32_t, rank C.uint32_t, info *C.daos_target_info_t, ev *C.struct_daos_event) C.int { + if daos_pool_query_target_RC != 0 { + return daos_pool_query_target_RC + } + + daos_pool_query_target_SetTgt = tgt + daos_pool_query_target_SetRank = rank + + info.ta_type = C.daos_target_type_t(daos_pool_query_target_Info.Type) + info.ta_state = C.daos_target_state_t(daos_pool_query_target_Info.State) + info.ta_space = daos_gds2cds(daos_pool_query_target_Info.Space) + + return daos_pool_query_target_RC +} + +var ( + daos_default_AttrList daos.AttributeList = daos.AttributeList{ + { + Name: "one", + Value: []byte("1"), + }, + { + Name: "two", + Value: []byte("2"), + }, + { + Name: "three", + Value: []byte("3"), + }, + } +) + +var ( + daos_pool_list_attr_AttrList daos.AttributeList = daos_default_AttrList + daos_pool_list_attr_CallCount int + daos_pool_list_attr_RCList []C.int + daos_pool_list_attr_RC C.int = 0 +) + +func reset_daos_pool_list_attr() { + daos_pool_list_attr_AttrList = daos_default_AttrList + daos_pool_list_attr_CallCount = 0 + daos_pool_list_attr_RCList = nil + daos_pool_list_attr_RC = 0 +} + +func daos_pool_list_attr(poolHdl C.daos_handle_t, buf *C.char, size *C.size_t, ev *C.struct_daos_event) C.int { + if len(daos_pool_list_attr_RCList) > 0 { + rc := daos_pool_list_attr_RCList[daos_pool_list_attr_CallCount] + daos_pool_list_attr_CallCount++ + if rc != 0 { + return rc + } + } + if daos_pool_list_attr_RC != 0 { + return daos_pool_list_attr_RC + } + + bufSize := 0 + for _, attr := range daos_pool_list_attr_AttrList { + bufSize += len(attr.Name) + 1 + } + *size = C.size_t(bufSize) + + if buf == nil { + return daos_pool_list_attr_RC + } + + bufSlice := unsafe.Slice((*C.char)(buf), bufSize) + bufPtr := 0 + for _, attr := range daos_pool_list_attr_AttrList { + for i := 0; i < len(attr.Name); i++ { + bufSlice[bufPtr] = C.char(attr.Name[i]) + bufPtr++ + } + bufSlice[bufPtr] = C.char(0) + bufPtr++ + } + + return daos_pool_list_attr_RC +} + +func daos_test_get_mappedNames(nameMap map[string]struct{}) []string { + names := make([]string, 0, len(nameMap)) + for name := range nameMap { + names = append(names, name) + } + return names +} + +var ( + daos_pool_get_attr_SetN int + daos_pool_get_attr_ReqNames map[string]struct{} + daos_pool_get_attr_CallCount int + daos_pool_get_attr_RCList []C.int + daos_pool_get_attr_AttrList daos.AttributeList = daos_default_AttrList + daos_pool_get_attr_RC C.int = 0 +) + +func reset_daos_pool_get_attr() { + daos_pool_get_attr_SetN = 0 + daos_pool_get_attr_ReqNames = nil + daos_pool_get_attr_CallCount = 0 + daos_pool_get_attr_RCList = nil + daos_pool_get_attr_AttrList = daos_default_AttrList + daos_pool_get_attr_RC = 0 +} + +func daos_pool_get_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + if len(daos_pool_get_attr_RCList) > 0 { + rc := daos_pool_get_attr_RCList[daos_pool_get_attr_CallCount] + daos_pool_get_attr_CallCount++ + if rc != 0 { + return rc + } + } + if daos_pool_get_attr_RC != 0 { + return daos_pool_get_attr_RC + } + + daos_pool_get_attr_SetN = int(n) + daos_pool_get_attr_ReqNames = make(map[string]struct{}) + cReqNames := unsafe.Slice(names, n) + for i := 0; i < int(n); i++ { + daos_pool_get_attr_ReqNames[C.GoString(cReqNames[i])] = struct{}{} + } + + if len(daos_pool_get_attr_ReqNames) > 0 && len(daos_pool_get_attr_AttrList) == 0 { + return -C.int(daos.Nonexistent) + } + + attrListMap := daos_pool_get_attr_AttrList.AsMap() + reqAttrCt := 0 + for attrName := range daos_pool_get_attr_ReqNames { + if _, ok := attrListMap[attrName]; !ok { + return -C.int(daos.Nonexistent) + } + reqAttrCt++ + } + + if reqAttrCt == 0 { + return daos_pool_get_attr_RC + } + + var valuesSlice []unsafe.Pointer + if values != nil { + valuesSlice = unsafe.Slice(values, reqAttrCt) + } + sizesSlice := unsafe.Slice(sizes, reqAttrCt) + idx := 0 + for _, attr := range daos_pool_get_attr_AttrList { + if _, ok := daos_pool_get_attr_ReqNames[attr.Name]; !ok { + continue + } + sizesSlice[idx] = C.size_t(len(attr.Value)) + if values != nil { + valSlice := unsafe.Slice((*byte)(valuesSlice[idx]), sizesSlice[idx]) + copy(valSlice[:], attr.Value) + } + idx++ + } + + return daos_pool_get_attr_RC +} + +var ( + daos_pool_set_attr_AttrList daos.AttributeList + daos_pool_set_attr_RC C.int = 0 +) + +func reset_daos_pool_set_attr() { + daos_pool_set_attr_AttrList = nil + daos_pool_set_attr_RC = 0 +} + +func daos_pool_set_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + if daos_pool_set_attr_RC != 0 { + return daos_pool_set_attr_RC + } + + namesSlice := unsafe.Slice(names, n) + valuesSlice := unsafe.Slice(values, n) + sizesSlice := unsafe.Slice(sizes, n) + for i := 0; i < int(n); i++ { + valueSlice := unsafe.Slice((*byte)(valuesSlice[i]), sizesSlice[i]) + daos_pool_set_attr_AttrList = append(daos_pool_set_attr_AttrList, &daos.Attribute{ + Name: C.GoString(namesSlice[i]), + Value: make([]byte, sizesSlice[i]), + }) + copy(daos_pool_set_attr_AttrList[len(daos_pool_set_attr_AttrList)-1].Value, valueSlice) + } + + return daos_pool_set_attr_RC +} + +var ( + daos_pool_del_attr_AttrNames []string + daos_pool_del_attr_RC C.int = 0 +) + +func reset_daos_pool_del_attr() { + daos_pool_del_attr_AttrNames = nil + daos_pool_del_attr_RC = 0 +} + +func daos_pool_del_attr(poolHdl C.daos_handle_t, n C.int, name **C.char, ev *C.struct_daos_event) C.int { + if daos_pool_del_attr_RC != 0 { + return daos_pool_del_attr_RC + } + + nameSlice := unsafe.Slice(name, n) + for i := 0; i < int(n); i++ { + daos_pool_del_attr_AttrNames = append(daos_pool_del_attr_AttrNames, C.GoString(nameSlice[i])) + } + + return daos_pool_del_attr_RC +} + +var ( + daos_mgmt_list_pools_SetSys string + daos_mgmt_list_pools_RetPools []*daos.PoolInfo = []*daos.PoolInfo{defaultPoolInfo()} + daos_mgmt_list_pools_CallCount int + daos_mgmt_list_pools_RCList []C.int + daos_mgmt_list_pools_RC C.int = 0 +) + +func reset_daos_mgmt_list_pools() { + daos_mgmt_list_pools_SetSys = "" + daos_mgmt_list_pools_RetPools = []*daos.PoolInfo{defaultPoolInfo()} + daos_mgmt_list_pools_CallCount = 0 + daos_mgmt_list_pools_RCList = nil + daos_mgmt_list_pools_RC = 0 +} + +func daos_mgmt_list_pools(sysName *C.char, poolCount *C.daos_size_t, pools *C.daos_mgmt_pool_info_t, ev *C.struct_daos_event) C.int { + if len(daos_mgmt_list_pools_RCList) > 0 { + rc := daos_mgmt_list_pools_RCList[daos_mgmt_list_pools_CallCount] + daos_mgmt_list_pools_CallCount++ + if rc != 0 { + return rc + } + } + if daos_mgmt_list_pools_RC != 0 { + return daos_mgmt_list_pools_RC + } + + *poolCount = C.daos_size_t(len(daos_mgmt_list_pools_RetPools)) + + daos_mgmt_list_pools_SetSys = C.GoString(sysName) + if *poolCount == 0 || pools == nil { + return daos_mgmt_list_pools_RC + } + + poolSlice := unsafe.Slice(pools, *poolCount) + for i, pool := range daos_mgmt_list_pools_RetPools { + poolSlice[i].mgpi_uuid = uuidToC(pool.UUID) + poolSlice[i].mgpi_label = C.CString(pool.Label) + poolSlice[i].mgpi_svc = ranklistFromGo(ranklist.RankSetFromRanks(pool.ServiceReplicas)) + poolSlice[i].mgpi_ldr = C.d_rank_t(pool.ServiceLeader) + } + + return daos_mgmt_list_pools_RC +} + +// ResetTestStubs will call the reset functions for all test stubs in order +// to reset state between tests. +func ResetTestStubs() { + reset_daos_pool_connect() + reset_daos_pool_query() + reset_daos_pool_query_target() + reset_daos_pool_list_attr() + reset_daos_pool_get_attr() + reset_daos_pool_set_attr() + reset_daos_pool_del_attr() + reset_daos_mgmt_list_pools() +} diff --git a/src/control/lib/daos/api/pool.go b/src/control/lib/daos/api/pool.go new file mode 100644 index 00000000000..f51d69a8814 --- /dev/null +++ b/src/control/lib/daos/api/pool.go @@ -0,0 +1,656 @@ +package api + +import ( + "context" + "unsafe" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +/* +#include +#include +#include + +#include "util.h" + +static inline uint32_t +get_rebuild_state(struct daos_rebuild_status *drs) +{ + if (drs == NULL) + return 0; + + return drs->rs_state; +} +*/ +import "C" + +type ( + // PoolHandle is an opaque type used to represent a DAOS Pool connection. + PoolHandle struct { + connHandle + } + + // ContainerHandle is an opaque type used to represent a DAOS Container connection. + // NB: A ContainerHandle contains the PoolHandle used to open the container. + ContainerHandle struct { + connHandle + PoolHandle *PoolHandle + } +) + +const ( + poolHandleKey ctxHdlKey = "poolHandle" + + MissingPoolLabel = "" +) + +// fromCtx retrieves the PoolHandle from the supplied context, if available. +func (ph *PoolHandle) fromCtx(ctx context.Context) error { + if ph == nil { + return ErrInvalidPoolHandle + } + return ph.connHandle.fromCtx(ctx, poolHandleKey) +} + +// toCtx returns a new context with the PoolHandle stashed in it. +func (ph *PoolHandle) toCtx(ctx context.Context) context.Context { + if ph == nil { + return ctx + } + return ph.connHandle.toCtx(ctx, poolHandleKey) +} + +// newPoolSpaceInfo constructs a Go type from the underlying C type. +func newPoolSpaceInfo(dps *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { + if dps == nil { + return nil + } + + return &daos.StorageUsageStats{ + Total: uint64(dps.ps_space.s_total[mt]), + Free: uint64(dps.ps_space.s_free[mt]), + Min: uint64(dps.ps_free_min[mt]), + Max: uint64(dps.ps_free_max[mt]), + Mean: uint64(dps.ps_free_mean[mt]), + MediaType: daos.StorageMediaType(mt), + } +} + +// newPoolRebuildStatus constructs a Go type from the underlying C type. +func newPoolRebuildStatus(drs *C.struct_daos_rebuild_status) *daos.PoolRebuildStatus { + if drs == nil { + return nil + } + + compatRebuildState := func() daos.PoolRebuildState { + switch { + case drs.rs_version == 0: + return daos.PoolRebuildStateIdle + case C.get_rebuild_state(drs) == C.DRS_COMPLETED: + return daos.PoolRebuildStateDone + default: + return daos.PoolRebuildStateBusy + } + } + + return &daos.PoolRebuildStatus{ + Status: int32(drs.rs_errno), + Objects: uint64(drs.rs_obj_nr), + Records: uint64(drs.rs_rec_nr), + State: compatRebuildState(), + } +} + +// newPoolInfo constructs a Go type from the underlying C type. +func newPoolInfo(cpi *C.daos_pool_info_t) *daos.PoolInfo { + if cpi == nil { + return nil + } + + poolInfo := new(daos.PoolInfo) + + poolInfo.QueryMask = daos.PoolQueryMask(cpi.pi_bits) + poolInfo.UUID = uuid.Must(uuidFromC(cpi.pi_uuid)) + poolInfo.TotalTargets = uint32(cpi.pi_ntargets) + poolInfo.DisabledTargets = uint32(cpi.pi_ndisabled) + poolInfo.ActiveTargets = uint32(cpi.pi_space.ps_ntargets) + poolInfo.TotalEngines = uint32(cpi.pi_nnodes) + poolInfo.ServiceLeader = uint32(cpi.pi_leader) + poolInfo.Version = uint32(cpi.pi_map_ver) + poolInfo.State = daos.PoolServiceStateReady + if poolInfo.DisabledTargets > 0 { + poolInfo.State = daos.PoolServiceStateDegraded + } + + poolInfo.Rebuild = newPoolRebuildStatus(&cpi.pi_rebuild_st) + if poolInfo.QueryMask.HasOption(daos.PoolQueryOptionSpace) { + poolInfo.TierStats = []*daos.StorageUsageStats{ + newPoolSpaceInfo(&cpi.pi_space, C.DAOS_MEDIA_SCM), + newPoolSpaceInfo(&cpi.pi_space, C.DAOS_MEDIA_NVME), + } + } + + return poolInfo +} + +func poolInfoFromProps(pi *daos.PoolInfo, propEntries []C.struct_daos_prop_entry) { + if pi == nil || len(propEntries) == 0 { + return + } + + for _, entry := range propEntries { + switch entry.dpe_type { + case C.DAOS_PROP_PO_LABEL: + pi.Label = C.GoString(C.get_dpe_str(&entry)) + case C.DAOS_PROP_PO_SVC_LIST: + rlPtr := C.get_dpe_val_ptr(&entry) + if rlPtr == nil { + return + } + rs, err := rankSetFromC((*C.d_rank_list_t)(rlPtr)) + if err != nil { + return + } + pi.ServiceReplicas = rs.Ranks() + } + } +} + +// Disconnect signals that the client no longer needs the DAOS pool +// connection and that it is safe to release resources allocated for +// the connection. +func (ph *PoolHandle) Disconnect(ctx context.Context) error { + if ph == nil { + return ErrInvalidPoolHandle + } + logging.FromContext(ctx).Debugf("Disconnect(%s)", ph) + + if err := daosError(daos_pool_disconnect(ph.daosHandle)); err != nil { + return errors.Wrap(err, "failed to disconnect from pool") + } + ph.invalidate() + + return nil +} + +// UUID returns the DAOS pool's UUID. +func (ph *PoolHandle) UUID() uuid.UUID { + if ph == nil { + return uuid.Nil + } + return ph.connHandle.UUID +} + +// Query is a convenience wrapper around the PoolQuery() function. +func (ph *PoolHandle) Query(ctx context.Context, mask daos.PoolQueryMask) (*daos.PoolInfo, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolQuery(ph.toCtx(ctx), "", mask) +} + +// QueryTargets is a convenience wrapper around the PoolQueryTargets() function. +func (ph *PoolHandle) QueryTargets(ctx context.Context, rank ranklist.Rank, targets *ranklist.RankSet) ([]*daos.PoolQueryTargetInfo, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolQueryTargets(ph.toCtx(ctx), "", rank, targets) +} + +// ListAttributes is a convenience wrapper around the PoolListAttributes() function. +func (ph *PoolHandle) ListAttributes(ctx context.Context) ([]string, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolListAttributes(ph.toCtx(ctx), "") +} + +// GetAttributes is a convenience wrapper around the PoolGetAttributes() function. +func (ph *PoolHandle) GetAttributes(ctx context.Context, attrNames ...string) (daos.AttributeList, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolGetAttributes(ph.toCtx(ctx), "", attrNames...) +} + +// SetAttributes is a convenience wrapper around the PoolSetAttributes() function. +func (ph *PoolHandle) SetAttributes(ctx context.Context, attrs ...*daos.Attribute) error { + if ph == nil { + return ErrInvalidPoolHandle + } + return PoolSetAttributes(ph.toCtx(ctx), "", attrs...) +} + +// DeleteAttributes is a convenience wrapper around the PoolDeleteAttributes() function. +func (ph *PoolHandle) DeleteAttributes(ctx context.Context, attrNames ...string) error { + if ph == nil { + return ErrInvalidPoolHandle + } + return PoolDeleteAttributes(ph.toCtx(ctx), "", attrNames...) +} + +type ( + // PoolConnectReq defines the parameters for a PoolConnect request. + PoolConnectReq struct { + ID string + SysName string + Flags daos.PoolConnectFlag + Query bool + } + + // PoolConnectResp contains the response to a PoolConnect request. + PoolConnectResp struct { + PoolConnection *PoolHandle + PoolInfo *daos.PoolInfo + } +) + +// PoolConnect establishes a connection to the specified DAOS pool. +// NB: The caller is responsible for disconnecting from the pool when +// finished. +func PoolConnect(ctx context.Context, req PoolConnectReq) (*PoolConnectResp, error) { + if ctx == nil { + return nil, errNilCtx + } + logging.FromContext(ctx).Debugf("PoolConnect(%+v)", req) + + if req.ID == "" { + return nil, errors.Wrap(daos.InvalidInput, "no pool ID provided") + } + if req.SysName == "" { + req.SysName = build.DefaultSystemName + } + if req.Flags == 0 { + req.Flags = daos.PoolConnectFlagReadOnly + } + + var dpi C.daos_pool_info_t + if req.Query { + dpi.pi_bits = C.ulong(daos.DefaultPoolQueryMask) + } + var poolConn PoolHandle + + cPoolID := C.CString(req.ID) + defer freeString(cPoolID) + cSys := C.CString(req.SysName) + defer freeString(cSys) + + if err := daosError(daos_pool_connect(cPoolID, cSys, C.uint(req.Flags), &poolConn.daosHandle, &dpi, nil)); err != nil { + return nil, errors.Wrap(err, "failed to connect to pool") + } + + poolInfo := newPoolInfo(&dpi) + poolConn.connHandle.UUID = poolInfo.UUID + if req.ID != poolInfo.UUID.String() { + poolInfo.Label = req.ID + } else { + // If the connection was made with a UUID, then we don't know the label without + // a query. This should be a rare scenario. If the request allows it, try a query. + poolInfo.Label = MissingPoolLabel + if req.Query { + qpi, err := poolConn.Query(ctx, daos.HealthOnlyPoolQueryMask) + if err != nil { + return nil, errors.Wrap(err, "failed to query pool for label") + } + poolInfo.Label = qpi.Label + } + } + // Set the label on the connection for convenience. + poolConn.connHandle.Label = poolInfo.Label + + return &PoolConnectResp{ + PoolConnection: &poolConn, + PoolInfo: poolInfo, + }, nil +} + +// getPoolConn retrieves the PoolHandle set in the context, if available, +// or tries to establish a new connection to the specified pool. +func getPoolConn(ctx context.Context, poolID string, flags daos.PoolConnectFlag) (*PoolHandle, func(), error) { + nulCleanup := func() {} + ph := &PoolHandle{} + if err := ph.fromCtx(ctx); err == nil { + if poolID != "" { + return nil, nulCleanup, errors.Wrap(daos.InvalidInput, "PoolHandle found in context with non-empty poolID") + } + return ph, nulCleanup, nil + } + + resp, err := PoolConnect(ctx, PoolConnectReq{ + ID: poolID, + Flags: flags, + Query: false, + }) + if err != nil { + return nil, nulCleanup, err + } + + cleanup := func() { + err := resp.PoolConnection.Disconnect(ctx) + if err != nil { + logging.FromContext(ctx).Error(err.Error()) + } + } + return resp.PoolConnection, cleanup, nil +} + +// PoolQuery retrieves information about the DAOS Pool, including health and rebuild status, +// storage usage, and other details. +func PoolQuery(ctx context.Context, poolID string, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { + if queryMask == 0 { + queryMask = daos.DefaultPoolQueryMask + } + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolQuery(%s:%s)", poolConn, queryMask) + + var enabledRanks *C.d_rank_list_t + var disabledRanks *C.d_rank_list_t + defer func() { + C.d_rank_list_free(enabledRanks) + C.d_rank_list_free(disabledRanks) + }() + + // Query for some additional information stored as properties. + queryProps := C.daos_prop_alloc(2) + propEntries := unsafe.Slice(queryProps.dpp_entries, queryProps.dpp_nr) + propEntries[0].dpe_type = C.DAOS_PROP_PO_LABEL + propEntries[1].dpe_type = C.DAOS_PROP_PO_SVC_LIST + defer func() { + C.daos_prop_free(queryProps) + }() + + var rc C.int + cPoolInfo := C.daos_pool_info_t{ + pi_bits: C.uint64_t(queryMask), + } + if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) && queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + enaQm := queryMask + enaQm.ClearOptions(daos.PoolQueryOptionDisabledEngines) + cPoolInfo.pi_bits = C.uint64_t(enaQm) + rc = daos_pool_query(poolConn.daosHandle, &enabledRanks, &cPoolInfo, queryProps, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to query pool") + } + + /* second query to just get disabled ranks */ + rc = daos_pool_query(poolConn.daosHandle, &disabledRanks, nil, nil, nil) + } else if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) { + rc = daos_pool_query(poolConn.daosHandle, &enabledRanks, &cPoolInfo, queryProps, nil) + } else if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + rc = daos_pool_query(poolConn.daosHandle, &disabledRanks, &cPoolInfo, queryProps, nil) + } else { + rc = daos_pool_query(poolConn.daosHandle, nil, &cPoolInfo, queryProps, nil) + } + + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to query pool") + } + + poolInfo := newPoolInfo(&cPoolInfo) + poolInfo.QueryMask = queryMask + poolInfoFromProps(poolInfo, propEntries) + + if enabledRanks != nil { + poolInfo.EnabledRanks, err = rankSetFromC(enabledRanks) + if err != nil { + return nil, err + } + } + if disabledRanks != nil { + poolInfo.DisabledRanks, err = rankSetFromC(disabledRanks) + if err != nil { + return nil, err + } + } + + return poolInfo, nil +} + +func convertPoolTargetInfo(ptinfo *C.daos_target_info_t) *daos.PoolQueryTargetInfo { + return &daos.PoolQueryTargetInfo{ + Type: daos.PoolQueryTargetType(ptinfo.ta_type), + State: daos.PoolQueryTargetState(ptinfo.ta_state), + Space: []*daos.StorageUsageStats{ + { + Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_SCM]), + Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_SCM]), + MediaType: C.DAOS_MEDIA_SCM, + }, + { + Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_NVME]), + Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_NVME]), + MediaType: C.DAOS_MEDIA_NVME, + }, + }, + } +} + +// PoolQueryTargets retrieves information about storage targets in the DAOS Pool. +func PoolQueryTargets(ctx context.Context, poolID string, rank ranklist.Rank, reqTargets *ranklist.RankSet) ([]*daos.PoolQueryTargetInfo, error) { + targets := ranklist.NewRankSet() + targets.Replace(reqTargets) + + if targets.Count() == 0 { + pi, err := PoolQuery(ctx, poolID, daos.HealthOnlyPoolQueryMask) + if err != nil || (pi.TotalTargets == 0 || pi.TotalEngines == 0) { + if err != nil { + return nil, errors.Wrap(err, "pool query failed") + } + return nil, errors.New("failed to derive target count from pool query") + } + tgtCount := pi.TotalTargets / pi.TotalEngines + for i := uint32(0); i < tgtCount; i++ { + targets.Add(ranklist.Rank(i)) + } + } + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolQueryTargets(%s:%d:[%s])", poolConn, rank, targets) + + ptInfo := C.daos_target_info_t{} + var rc C.int + + infos := make([]*daos.PoolQueryTargetInfo, 0, targets.Count()) + for _, tgt := range targets.Ranks() { + rc = daos_pool_query_target(poolConn.daosHandle, C.uint32_t(tgt), C.uint32_t(rank), &ptInfo, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to query pool %s rank:target %d:%d", poolID, rank, tgt) + } + + infos = append(infos, convertPoolTargetInfo(&ptInfo)) + } + + return infos, nil +} + +// PoolListAttributes returns a list of user-definable pool attribute names. +func PoolListAttributes(ctx context.Context, poolID string) ([]string, error) { + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolListAttributes(%s)", poolConn) + + if err := ctx.Err(); err != nil { + return nil, ctxErr(err) + } + + return listDaosAttributes(poolConn.daosHandle, poolAttr) +} + +// PoolGetAttributes fetches the specified pool attributes. If no +// attribute names are provided, all attributes are fetched. +func PoolGetAttributes(ctx context.Context, poolID string, names ...string) (daos.AttributeList, error) { + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolGetAttributes(%s:%v)", poolConn, names) + + if err := ctx.Err(); err != nil { + return nil, ctxErr(err) + } + + return getDaosAttributes(poolConn.daosHandle, poolAttr, names) +} + +// PoolSetAttributes sets the specified pool attributes. +func PoolSetAttributes(ctx context.Context, poolID string, attrs ...*daos.Attribute) error { + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolSetAttributes(%s:%v)", poolConn, attrs) + + if err := ctx.Err(); err != nil { + return ctxErr(err) + } + + return setDaosAttributes(poolConn.daosHandle, poolAttr, attrs) +} + +// PoolDeleteAttributes deletes the specified pool attributes. +func PoolDeleteAttributes(ctx context.Context, poolID string, attrNames ...string) error { + poolConn, disconnect, err := getPoolConn(ctx, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolDeleteAttributes(%s:%+v)", poolConn, attrNames) + + if err := ctx.Err(); err != nil { + return ctxErr(err) + } + + return delDaosAttributes(poolConn.daosHandle, poolAttr, attrNames) +} + +type ( + // GetPoolListReq defines the parameters for a GetPoolList request. + GetPoolListReq struct { + SysName string + Query bool + } +) + +// GetPoolList returns a list of DAOS pools in the system. +func GetPoolList(ctx context.Context, req GetPoolListReq) ([]*daos.PoolInfo, error) { + log := logging.FromContext(ctx) + log.Debugf("GetPoolList(%+v)", req) + + if ctx == nil { + return nil, errNilCtx + } + if req.SysName == "" { + req.SysName = build.DefaultSystemName + } + cSysName := C.CString(req.SysName) + defer freeString(cSysName) + + var cPools []C.daos_mgmt_pool_info_t + for { + var rc C.int + var poolCount C.size_t + + // First, fetch the total number of pools in the system. + // We may not have access to all of them, so this is an upper bound. + rc = daos_mgmt_list_pools(cSysName, &poolCount, nil, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to list pools") + } + log.Debugf("pools in system: %d", poolCount) + + if poolCount < 1 { + return nil, nil + } + + // Now, fetch the pools into a buffer sized for the number of pools found. + cPools = make([]C.daos_mgmt_pool_info_t, poolCount) + rc = daos_mgmt_list_pools(cSysName, &poolCount, &cPools[0], nil) + err := daosError(rc) + if err == nil { + cPools = cPools[:poolCount] // adjust the slice to the number of pools retrieved + log.Debugf("fetched %d pools", len(cPools)) + break + } + if err == daos.StructTooSmall { + log.Notice("server-side pool list changed; re-fetching") + continue + } + log.Errorf("failed to fetch pool list: %s", err) + return nil, errors.Wrap(err, "failed to list pools") + } + + pools := make([]*daos.PoolInfo, 0, len(cPools)) + for i := 0; i < len(cPools); i++ { + cPool := &cPools[i] + + svcLdr := uint32(cPool.mgpi_ldr) + svcRanks, err := rankSetFromC(cPool.mgpi_svc) + if err != nil { + return nil, err + } + defer func() { + C.d_rank_list_free(cPool.mgpi_svc) + }() + poolUUID, err := uuidFromC(cPool.mgpi_uuid) + if err != nil { + return nil, err + } + poolLabel := C.GoString(cPool.mgpi_label) + + var pool *daos.PoolInfo + if req.Query { + pcResp, err := PoolConnect(ctx, PoolConnectReq{ + ID: poolUUID.String(), + SysName: req.SysName, + Flags: daos.PoolConnectFlagReadOnly, + Query: true, + }) + if err != nil { + log.Errorf("failed to connect to pool %q: %s", poolLabel, err) + continue + } + if err := pcResp.PoolConnection.Disconnect(ctx); err != nil { + log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) + } + pool = pcResp.PoolInfo + + // Add a few missing pieces that the query doesn't fill in. + pool.Label = poolLabel + pool.ServiceLeader = svcLdr + pool.ServiceReplicas = svcRanks.Ranks() + } else { + // Just populate the basic info. + pool = &daos.PoolInfo{ + UUID: poolUUID, + Label: poolLabel, + ServiceLeader: svcLdr, + ServiceReplicas: svcRanks.Ranks(), + State: daos.PoolServiceStateReady, + } + } + + pools = append(pools, pool) + } + + log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) + return pools, nil +} diff --git a/src/control/lib/daos/api/pool_test.go b/src/control/lib/daos/api/pool_test.go new file mode 100644 index 00000000000..5d2a15cff2b --- /dev/null +++ b/src/control/lib/daos/api/pool_test.go @@ -0,0 +1,1053 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +func TestAPI_PoolConnect(t *testing.T) { + defaultReq := PoolConnectReq{ + ID: daos_default_PoolInfo.Label, + SysName: build.DefaultSystemName, + Flags: daos.PoolConnectFlagReadWrite, + Query: true, + } + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + connReq PoolConnectReq + checkParams func(t *testing.T) + expResp *PoolConnectResp + expErr error + }{ + "nil context": { + connReq: defaultReq, + expErr: errNilCtx, + }, + "no poolID in req": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: defaultReq.Query, + }, + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_connect() fails": { + setup: func(t *testing.T) { + daos_pool_connect_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + connReq: defaultReq, + expErr: errors.Wrap(daos.IOError, "failed to connect to pool"), + }, + "daos_pool_connect() succeeds": { + ctx: test.Context(t), + connReq: defaultReq, + checkParams: func(t *testing.T) { + test.CmpAny(t, "poolID", defaultReq.ID, daos_pool_connect_SetPoolID) + test.CmpAny(t, "sysName", defaultReq.SysName, daos_pool_connect_SetSys) + test.CmpAny(t, "flags", defaultReq.Flags, daos_pool_connect_SetFlags) + test.CmpAny(t, "query", daos.DefaultPoolQueryMask, daos_pool_connect_QueryMask) + }, + expResp: &PoolConnectResp{ + PoolConnection: &PoolHandle{ + connHandle: connHandle{ + Label: daos_default_PoolInfo.Label, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + PoolInfo: defaultPoolInfo(), + }, + }, + "Connect with UUID and query enabled": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: true, + }, + expResp: &PoolConnectResp{ + PoolConnection: &PoolHandle{ + connHandle: connHandle{ + Label: daos_default_PoolInfo.Label, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + PoolInfo: defaultPoolInfo(), + }, + }, + "Connect with UUID and query enabled -- query fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: true, + }, + expErr: daos.IOError, + }, + "Connect with UUID and query disabled": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: false, + }, + expResp: &PoolConnectResp{ + PoolConnection: &PoolHandle{ + connHandle: connHandle{ + Label: MissingPoolLabel, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + PoolInfo: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.Label = MissingPoolLabel + return out + }(), + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, gotErr := PoolConnect(mustLogCtx(tc.ctx, log), tc.connReq) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b *PoolHandle) bool { + return a != nil && b != nil && a.String() == b.String() + }), + // These fields aren't populated in the PoolConnect() query. + cmpopts.IgnoreFields(daos.PoolInfo{}, + "EnabledRanks", "DisabledRanks", "DeadRanks", "ServiceReplicas", + ), + } + test.CmpAny(t, "PoolConnectResp", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +var ( + testCtxHandle = &PoolHandle{ + connHandle: connHandle{ + UUID: test.MockPoolUUID(43), + Label: "test-ctx", + }, + } + + testConnHandle = &PoolHandle{ + connHandle: connHandle{ + daosHandle: daos_default_pool_connect_Handle, + UUID: daos_default_PoolInfo.UUID, + Label: daos_default_PoolInfo.Label, + }, + } +) + +func TestAPI_getPoolConn(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + flags daos.PoolConnectFlag + checkParams func(t *testing.T) + expHdl *PoolHandle + expErr error + }{ + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle in context": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expHdl: testCtxHandle, + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "pool handle from Connect()": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + checkParams: func(t *testing.T) { + test.CmpAny(t, "poolID", daos_default_PoolInfo.Label, daos_pool_connect_SetPoolID) + test.CmpAny(t, "sysName", build.DefaultSystemName, daos_pool_connect_SetSys) + test.CmpAny(t, "flags", daos.PoolConnectFlagReadOnly, daos_pool_connect_SetFlags) + test.CmpAny(t, "query", daos.PoolQueryMask(0), daos_pool_connect_QueryMask) + }, + expHdl: testConnHandle, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + ctx := tc.ctx + if ctx == nil { + ctx = test.Context(t) + } + + ph, cleanup, gotErr := getPoolConn(mustLogCtx(ctx, log), tc.poolID, tc.flags) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + t.Cleanup(cleanup) + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b *PoolHandle) bool { + return a != nil && b != nil && a.String() == b.String() + }), + } + test.CmpAny(t, "PoolHandle", tc.expHdl, ph, cmpOpts...) + }) + } +} + +func TestAPI_PoolQuery(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + queryMask daos.PoolQueryMask + checkParams func(t *testing.T) + expResp *daos.PoolInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_query() fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "daos_pool_query() fails on enabled ranks": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines), + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "unspecified query mask": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.DefaultPoolQueryMask + out.EnabledRanks = nil + return out + }(), + }, + "default query mask": { + ctx: testCtxHandle.toCtx(test.Context(t)), + queryMask: daos.DefaultPoolQueryMask, + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.DefaultPoolQueryMask + out.EnabledRanks = nil + return out + }(), + }, + "health-only query mask": { + ctx: testCtxHandle.toCtx(test.Context(t)), + queryMask: daos.HealthOnlyPoolQueryMask, + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.HealthOnlyPoolQueryMask + out.EnabledRanks = nil + out.TierStats = nil + return out + }(), + }, + "enabled ranks": { + ctx: testCtxHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines) + out.DisabledRanks = nil + out.TierStats = nil + return out + }(), + }, + "enabled & disabled ranks": { + ctx: testCtxHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) + out.TierStats = nil + return out + }(), + }, + "space-only": { + ctx: testCtxHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionSpace), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionSpace) + out.EnabledRanks = nil + out.DisabledRanks = nil + return out + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, err := PoolQuery(mustLogCtx(tc.ctx, log), tc.poolID, tc.queryMask) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + } + test.CmpAny(t, "PoolQuery() PoolInfo", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +func TestAPI_PoolQueryTargets(t *testing.T) { + allTgtCt := daos_default_PoolInfo.TotalTargets / daos_default_PoolInfo.TotalEngines + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + rank ranklist.Rank + targets *ranklist.RankSet + checkParams func(t *testing.T) + expResp []*daos.PoolQueryTargetInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_query() fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "daos_pool_query_target() fails": { + setup: func(t *testing.T) { + daos_pool_query_target_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: daos.IOError, + }, + "pool query returns zero targets": { + setup: func(t *testing.T) { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_PoolInfo.TotalTargets = 0 + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.New("failed to derive target count"), + }, + "pool query returns zero engines": { + setup: func(t *testing.T) { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_PoolInfo.TotalEngines = 0 + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.New("failed to derive target count"), + }, + "nil target set gets all": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: nil, + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(allTgtCt-1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, allTgtCt) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + "empty target set gets all": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: ranklist.NewRankSet(), + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(allTgtCt-1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, allTgtCt) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + "specified target should not query pool for target list": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) // fail if the pool is queried + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: ranklist.MustCreateRankSet("1"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, 1) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, err := PoolQueryTargets(mustLogCtx(tc.ctx, log), tc.poolID, tc.rank, tc.targets) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + } + test.CmpAny(t, "PoolQueryTargets() response", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +func TestAPI_PoolListAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + expNames []string + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_list_attr() fails (get buf size)": { + setup: func(t *testing.T) { + daos_pool_list_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list attributes"), + }, + "daos_pool_list_attr() fails (fetch names)": { + setup: func(t *testing.T) { + daos_pool_list_attr_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.IOError), + } + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list attributes"), + }, + "no attributes set": { + setup: func(t *testing.T) { + daos_pool_list_attr_AttrList = nil + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + }, + "success": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expNames: []string{ + daos_default_AttrList[0].Name, + daos_default_AttrList[1].Name, + daos_default_AttrList[2].Name, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotNames, err := PoolListAttributes(mustLogCtx(tc.ctx, log), tc.poolID) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolListAttributes()", tc.expNames, gotNames) + }) + } +} + +func TestAPI_PoolGetAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + attrNames []string + checkParams func(t *testing.T) + expAttrs daos.AttributeList + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_list_attr() fails": { + setup: func(t *testing.T) { + daos_pool_list_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list attributes"), + }, + "daos_pool_get_attr() fails (sizes)": { + setup: func(t *testing.T) { + daos_pool_get_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to get attribute sizes"), + }, + "daos_pool_get_attr() fails (values)": { + setup: func(t *testing.T) { + daos_pool_get_attr_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.IOError), + } + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to get attribute values"), + }, + "empty requested attribute name": { + ctx: testCtxHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "a", ""), + expErr: errors.Errorf("empty attribute name at index 1"), + }, + "no attributes set; attributes requested": { + setup: func(t *testing.T) { + daos_pool_get_attr_AttrList = nil + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "foo"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "req attr names", test.JoinArgs(nil, "foo"), daos_pool_get_attr_ReqNames) + }, + expErr: errors.Wrap(daos.Nonexistent, "failed to get attribute sizes"), + }, + "unknown attribute requested": { + ctx: testCtxHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "foo"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "req attr names", test.JoinArgs(nil, "foo"), daos_pool_get_attr_ReqNames) + }, + expErr: errors.Wrap(daos.Nonexistent, "failed to get attribute sizes"), + }, + "no attributes set; no attributes requested": { + setup: func(t *testing.T) { + daos_pool_list_attr_AttrList = nil + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + }, + "success; all attributes": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expAttrs: daos_default_AttrList, + }, + "success; requested attributes": { + ctx: testCtxHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, daos_default_AttrList[0].Name, daos_default_AttrList[2].Name), + checkParams: func(t *testing.T) { + reqNames := test.JoinArgs(nil, daos_default_AttrList[0].Name, daos_default_AttrList[2].Name) + sort.Strings(reqNames) + gotNames := daos_test_get_mappedNames(daos_pool_get_attr_ReqNames) + sort.Strings(gotNames) + test.CmpAny(t, "req attr names", reqNames, gotNames) + }, + expAttrs: daos.AttributeList{ + daos_default_AttrList[0], + daos_default_AttrList[2], + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotAttrs, err := PoolGetAttributes(mustLogCtx(tc.ctx, log), tc.poolID, tc.attrNames...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + test.CmpAny(t, "PoolGetAttributes() daos.AttributeList", tc.expAttrs, gotAttrs) + }) + } +} + +func TestAPI_PoolSetAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + toSet daos.AttributeList + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "no attributes to set": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.InvalidInput, "no attributes provided"), + }, + "nil toSet attribute": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, nil), + expErr: errors.Wrap(daos.InvalidInput, "nil attribute at index 3"), + }, + "toSet attribute with empty name": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, &daos.Attribute{Name: ""}), + expErr: errors.Wrap(daos.InvalidInput, "empty attribute name at index 3"), + }, + "toSet attribute with empty value": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, &daos.Attribute{Name: "empty"}), + expErr: errors.Wrap(daos.InvalidInput, "empty attribute value at index 3"), + }, + "daos_pool_set_attr() fails": { + setup: func(t *testing.T) { + daos_pool_set_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + toSet: daos_default_AttrList, + expErr: errors.Wrap(daos.IOError, "failed to set attributes"), + }, + "success": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toSet: daos_default_AttrList, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + err := PoolSetAttributes(mustLogCtx(tc.ctx, log), tc.poolID, tc.toSet...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolSetAttributes() daos.AttributeList", tc.toSet, daos_pool_set_attr_AttrList) + }) + } +} + +func TestAPI_PoolDeleteAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + toDelete []string + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxHandle.toCtx(test.Context(t)), + poolID: "test", + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "no attributes to delete": { + ctx: testCtxHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.InvalidInput, "no attribute names provided"), + }, + "empty name in toDelete list": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, "foo", "", "bar"), + expErr: errors.Wrap(daos.InvalidInput, "empty attribute name at index 1"), + }, + "daos_pool_det_attr() fails": { + setup: func(t *testing.T) { + daos_pool_del_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, daos_default_AttrList[0].Name), + expErr: errors.Wrap(daos.IOError, "failed to delete attributes"), + }, + "success": { + ctx: testCtxHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, daos_default_AttrList[0].Name), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + err := PoolDeleteAttributes(mustLogCtx(tc.ctx, log), tc.poolID, tc.toDelete...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolDeleteAttributes() AttrNames", tc.toDelete, daos_pool_del_attr_AttrNames) + }) + } +} + +func TestAPI_PoolHandleMethods(t *testing.T) { + testHandle := &PoolHandle{} + + thType := reflect.TypeOf(testHandle) + for i := 0; i < thType.NumMethod(); i++ { + method := thType.Method(i) + methArgs := make([]reflect.Value, 0) + var expResults int + + switch method.Name { + case "Disconnect": + expResults = 1 + case "Query": + methArgs = append(methArgs, reflect.ValueOf(daos.DefaultPoolQueryMask)) + expResults = 2 + case "QueryTargets": + methArgs = append(methArgs, reflect.ValueOf(ranklist.Rank(1)), reflect.ValueOf((*ranklist.RankSet)(nil))) + expResults = 2 + case "ListAttributes": + expResults = 2 + case "GetAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0].Name)) + expResults = 2 + case "SetAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0])) + expResults = 1 + case "DeleteAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0].Name)) + expResults = 1 + case "FillHandle", "IsValid", "String", "UUID": + // No tests for these. The main point of this suite is to ensure that the + // convenience wrappers handle inputs as expected. + continue + default: + // If you're here, you need to add a case to test your new method. + t.Fatalf("unhandled method %q", method.Name) + } + + // Not intended to be exhaustive; just verify that they accept the parameters + // we expect and return something sensible for errors. + for name, tc := range map[string]struct { + setup func(t *testing.T) + th *PoolHandle + expErr error + }{ + fmt.Sprintf("%s: nil handle", method.Name): { + th: nil, + expErr: ErrInvalidPoolHandle, + }, + fmt.Sprintf("%s: success", method.Name): { + th: testHandle, + }, + } { + t.Run(name, func(t *testing.T) { + thArg := reflect.ValueOf(tc.th) + if tc.th == nil { + thArg = reflect.New(thType).Elem() + } + ctxArg := reflect.ValueOf(test.Context(t)) + testArgs := append([]reflect.Value{thArg, ctxArg}, methArgs...) + t.Logf("\nargs: %+v", testArgs) + + retVals := method.Func.Call(testArgs) + if len(retVals) != expResults { + t.Fatalf("expected %d return values, got %d", expResults, len(retVals)) + } + + if err, ok := retVals[len(retVals)-1].Interface().(error); ok { + test.CmpErr(t, tc.expErr, err) + } else { + test.CmpErr(t, tc.expErr, nil) + } + }) + } + } +} + +func TestAPI_GetPoolList(t *testing.T) { + defaultReq := GetPoolListReq{ + SysName: "non-default", + Query: true, + } + defaultPoolInfoResp := []*daos.PoolInfo{ + { + State: daos.PoolServiceStateReady, + UUID: daos_default_PoolInfo.UUID, + Label: daos_default_PoolInfo.Label, + ServiceReplicas: daos_default_PoolInfo.ServiceReplicas, + ServiceLeader: daos_default_PoolInfo.ServiceLeader, + }, + } + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + req GetPoolListReq + checkParams func(t *testing.T) + expPools []*daos.PoolInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "daos_mgmt_list_pools fails (sizes)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + expErr: errors.Wrap(daos.IOError, "failed to list pools"), + }, + "daos_mgmt_list_pools fetch fails (not retryable)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.NoMemory), + } + }, + ctx: test.Context(t), + expErr: errors.Wrap(daos.NoMemory, "failed to list pools"), + }, + "daos_pool_connect fails": { + setup: func(t *testing.T) { + daos_pool_connect_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + req: defaultReq, + expPools: []*daos.PoolInfo{}, + }, + "daos_mgmt_list_pools fetch fails (retryable)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.StructTooSmall), + 0, + 0, + } + }, + ctx: test.Context(t), + expPools: defaultPoolInfoResp, + }, + "default system name supplied": { + ctx: test.Context(t), + req: GetPoolListReq{}, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", build.DefaultSystemName, daos_mgmt_list_pools_SetSys) + }, + expPools: defaultPoolInfoResp, + }, + "success (no pools)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RetPools = nil + }, + ctx: test.Context(t), + req: defaultReq, + }, + "success (no query)": { + ctx: test.Context(t), + req: GetPoolListReq{ + SysName: defaultReq.SysName, + }, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", defaultReq.SysName, daos_mgmt_list_pools_SetSys) + }, + expPools: defaultPoolInfoResp, + }, + "success (query)": { + ctx: test.Context(t), + req: defaultReq, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", defaultReq.SysName, daos_mgmt_list_pools_SetSys) + }, + expPools: func() []*daos.PoolInfo { + pi := copyPoolInfo(&daos_default_PoolInfo) + pi.EnabledRanks = nil + pi.DisabledRanks = nil + + return []*daos.PoolInfo{pi} + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotPools, err := GetPoolList(mustLogCtx(tc.ctx, log), tc.req) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + test.CmpAny(t, "GetPoolList() PoolList", tc.expPools, gotPools) + }) + } +} diff --git a/src/control/lib/daos/api/util.go b/src/control/lib/daos/api/util.go new file mode 100644 index 00000000000..9c461cd5c16 --- /dev/null +++ b/src/control/lib/daos/api/util.go @@ -0,0 +1,117 @@ +package api + +import ( + "context" + "unsafe" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +/* +#include +#include + +#include + +#include "util.h" +*/ +import "C" + +func goBool2int(in bool) (out C.int) { + if in { + out = 1 + } + return +} + +func copyUUID(dst *C.uuid_t, src uuid.UUID) error { + if dst == nil { + return errors.Wrap(daos.InvalidInput, "nil dest uuid_t") + } + + for i, v := range src { + dst[i] = C.uchar(v) + } + + return nil +} + +func uuidToC(in uuid.UUID) (out C.uuid_t) { + for i, v := range in { + out[i] = C.uchar(v) + } + + return +} + +func uuidFromC(cUUID C.uuid_t) (uuid.UUID, error) { + return uuid.FromBytes(C.GoBytes(unsafe.Pointer(&cUUID[0]), C.int(len(cUUID)))) +} + +func freeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} + +func iterStringsBuf(cBuf unsafe.Pointer, expected C.size_t, cb func(string)) error { + var curLen C.size_t + + // Create a Go slice for easy iteration (no pointer arithmetic in Go). + bufSlice := unsafe.Slice((*C.char)(cBuf), expected) + for total := C.size_t(0); total < expected; total += curLen + 1 { + chunk := bufSlice[total:] + curLen = C.strnlen(&chunk[0], expected-total) + + if curLen >= expected-total { + return errors.Wrap(daos.NoMemory, "corrupt buffer") + } + + chunk = bufSlice[total : total+curLen] + cb(C.GoString(&chunk[0])) + } + + return nil +} + +func rankSetFromC(cRankList *C.d_rank_list_t) (*ranklist.RankSet, error) { + if cRankList == nil { + return nil, errors.Wrap(daos.InvalidInput, "nil ranklist") + } + + cRankSlice := unsafe.Slice(cRankList.rl_ranks, cRankList.rl_nr) + rs := ranklist.NewRankSet() + for _, cRank := range cRankSlice { + rs.Add(ranklist.Rank(cRank)) + } + + return rs, nil +} + +func ranklistFromGo(rs *ranklist.RankSet) *C.d_rank_list_t { + if rs == nil { + return nil + } + + rl := C.d_rank_list_alloc(C.uint32_t(rs.Count())) + cRanks := unsafe.Slice(rl.rl_ranks, rs.Count()) + for i, r := range rs.Ranks() { + cRanks[i] = C.d_rank_t(r) + } + + return rl +} + +func mustLogCtx(parent context.Context, log logging.Logger) context.Context { + if parent == nil { + return nil + } + ctx, err := logging.ToContext(parent, log) + if err != nil { + panic(err) + } + return ctx +} diff --git a/src/control/lib/daos/api/util.h b/src/control/lib/daos/api/util.h new file mode 100644 index 00000000000..d0ddb067ca2 --- /dev/null +++ b/src/control/lib/daos/api/util.h @@ -0,0 +1,101 @@ +#ifndef __DAOS_API_UTIL_H__ +#define __DAOS_API_UTIL_H__ + +// #define D_LOGFAC DD_FAC(client) +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include + +/* cgo is unable to work directly with preprocessor macros + * so we have to provide these glue helpers. + */ +static inline uint64_t +daos_prop_co_status_val(uint32_t status, uint32_t flag, uint32_t ver) +{ + return DAOS_PROP_CO_STATUS_VAL(status, flag, ver); +} + +static void +daos_free(void *ptr) +{ + D_FREE(ptr); +} + +/* cgo is unable to work directly with unions, so we have + * to provide these glue helpers. + */ +static inline char * +get_dpe_str(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return NULL; + + return dpe->dpe_str; +} + +static inline uint64_t +get_dpe_val(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return 0; + + return dpe->dpe_val; +} + +static inline void * +get_dpe_val_ptr(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return NULL; + + return dpe->dpe_val_ptr; +} + +static inline bool +dpe_is_negative(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return 0; + + return dpe->dpe_flags & DAOS_PROP_ENTRY_NOT_SET; +} + +static inline void +set_dpe_str(struct daos_prop_entry *dpe, d_string_t str) +{ + if (dpe == NULL) + return; + + dpe->dpe_str = str; +} + +static inline void +set_dpe_val(struct daos_prop_entry *dpe, uint64_t val) +{ + if (dpe == NULL) + return; + + dpe->dpe_val = val; +} + +static inline void +set_dpe_val_ptr(struct daos_prop_entry *dpe, void *val_ptr) +{ + if (dpe == NULL) + return; + + dpe->dpe_val_ptr = val_ptr; +} + +/*static inline uint32_t +get_rebuild_state(struct daos_rebuild_status *drs) +{ + if (drs == NULL) + return 0; + + return drs->rs_state; +}*/ + +#endif /* __DAOS_API_UTIL_H__ */ \ No newline at end of file diff --git a/src/control/lib/daos/attribute.go b/src/control/lib/daos/attribute.go new file mode 100644 index 00000000000..dcd9873efc4 --- /dev/null +++ b/src/control/lib/daos/attribute.go @@ -0,0 +1,39 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import "sort" + +type ( + // Attribute is a pool or container attribute. + Attribute struct { + Name string `json:"name"` + Value []byte `json:"value,omitempty"` + } + + // AttributeList is a list of attributes. + AttributeList []*Attribute +) + +// AsMap returns the attributes list as a map. +func (al AttributeList) AsMap() map[string][]byte { + m := make(map[string][]byte) + for _, a := range al { + m[a.Name] = a.Value + } + return m +} + +// AsList returns the attributes list as a sorted list of attribute names. +func (al AttributeList) AsList() []string { + names := make([]string, len(al)) + for i, a := range al { + names[i] = a.Name + } + sort.Strings(names) + return names +} diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go index 6555a8cf721..8be88af45fe 100644 --- a/src/control/lib/daos/pool.go +++ b/src/control/lib/daos/pool.go @@ -106,6 +106,9 @@ type ( // PoolQueryMask implements a bitmask for pool query options. PoolQueryMask C.uint64_t + + // PoolConnectFlag represents DAOS pool connect options. + PoolConnectFlag uint ) const ( @@ -126,11 +129,11 @@ const ( PoolQueryOptionDeadEngines PoolQueryOption = "dead_engines" // PoolConnectFlagReadOnly indicates that the connection is read-only. - PoolConnectFlagReadOnly = C.DAOS_PC_RO + PoolConnectFlagReadOnly PoolConnectFlag = C.DAOS_PC_RO // PoolConnectFlagReadWrite indicates that the connection is read-write. - PoolConnectFlagReadWrite = C.DAOS_PC_RW + PoolConnectFlagReadWrite PoolConnectFlag = C.DAOS_PC_RW // PoolConnectFlagExclusive indicates that the connection is exclusive. - PoolConnectFlagExclusive = C.DAOS_PC_EX + PoolConnectFlagExclusive PoolConnectFlag = C.DAOS_PC_EX ) func (pqo PoolQueryOption) String() string { diff --git a/src/control/lib/daos/util.go b/src/control/lib/daos/util.go new file mode 100644 index 00000000000..ebc617a4d64 --- /dev/null +++ b/src/control/lib/daos/util.go @@ -0,0 +1,18 @@ +// +// (C) Copyright 2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import "unsafe" + +/* +#include +*/ +import "C" + +func freeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} diff --git a/src/control/lib/ranklist/ranklist.go b/src/control/lib/ranklist/ranklist.go index c65d3e7259f..230a8b20c33 100644 --- a/src/control/lib/ranklist/ranklist.go +++ b/src/control/lib/ranklist/ranklist.go @@ -98,7 +98,7 @@ func (rs *RankSet) Merge(other *RankSet) { // Replace replaces the contents of the receiver with the supplied RankSet. func (rs *RankSet) Replace(other *RankSet) { - if rs == nil || other == nil { + if rs == nil || other == nil || other.ns == nil { return }