Skip to content

Commit

Permalink
Checkpoint hardware pool work. (#86)
Browse files Browse the repository at this point in the history
* Also fix an off-by-one error in qvi_hwloc_bitmap_nbits().
* Checkpoint some work looking into a potentially nicer way to deal with
  C++ objects.

Signed-off-by: Samuel K. Gutierrez <samuel@lanl.gov>
  • Loading branch information
samuelkgutierrez authored Mar 14, 2024
1 parent 2550431 commit 2881434
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 43 deletions.
4 changes: 2 additions & 2 deletions src/qvi-hwloc.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2020-2023 Triad National Security, LLC
* Copyright (c) 2020-2024 Triad National Security, LLC
* All rights reserved.
*
* Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC
Expand Down Expand Up @@ -804,7 +804,7 @@ qvi_hwloc_bitmap_nbits(
const int inbits = hwloc_bitmap_last(cpuset);
if (inbits == -1) return QV_ERR_HWLOC;

*nbits = (size_t)inbits;
*nbits = size_t(inbits) + 1;
return QV_SUCCESS;
}

Expand Down
128 changes: 88 additions & 40 deletions src/qvi-hwpool.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2022 Triad National Security, LLC
* Copyright (c) 2022-2024 Triad National Security, LLC
* All rights reserved.
*
* This file is part of the quo-vadis project. See the LICENSE file at the
Expand Down Expand Up @@ -48,39 +48,90 @@
#include "qvi-hwloc.h"
#include "qvi-utils.h"

struct qvi_hwpool_s {
/** The cpuset of this resource pool. */
/**
* Maintains a mapping between a resource ID and associated hint flags.
*/
using qvi_hwpool_resource_id_hint_map_t = std::unordered_map<
uint32_t, qv_scope_create_hint_t
>;

/**
* Maintains information about a pool of hardware resources by resource ID.
*/
struct qvi_hwpool_resource_info_s {
/**
* Vector maintaining the reference count for a given resource ID. There is
* a one-to-one correspondence between resource IDs and addressable vector
* slots since every resource is reference counted. For example, each bit in
* a cpuset will have a corresponding reference count indexed by the bit's
* location in the cpuset.
*/
std::vector<uint32_t> resid_ref_count;
qvi_hwpool_resource_id_hint_map_t resid_hint_map;
};

/**
* Base hardware pool resource class.
*/
struct qvi_hwpool_resource_s {
/** Resource info. */
qvi_hwpool_resource_info_s resinfo;
/** Base constructor that does minimal work. */
qvi_hwpool_resource_s(void) = default;
/** Virtual destructor. */
virtual ~qvi_hwpool_resource_s(void) = default;
};

/**
* Defines a cpuset pool.
*/
struct qvi_hwpool_cpus_s : qvi_hwpool_resource_s {
int qvim_rc = QV_ERR_INTERNAL;
/** The cpuset of the maintained CPUs. */
hwloc_bitmap_t cpuset = nullptr;

qvi_hwpool_cpus_s(void)
{
qvim_rc = qvi_hwloc_bitmap_calloc(&cpuset);
}

virtual
~qvi_hwpool_cpus_s(void)
{
qvi_hwloc_bitmap_free(&cpuset);
}
};

struct qvi_hwpool_s {
int qvim_rc = QV_ERR_INTERNAL;
/** The CPUs in the resource pool. */
qvi_hwpool_cpus_s cpus;
/** Device information. */
qvi_hwpool_devinfos_t *devinfos = nullptr;
qvi_hwpool_devinfos_t devinfos;
// TODO(skg) Add owner to structure?
/** The obtained cpuset of this resource pool. */
hwloc_bitmap_t obcpuset = nullptr;

qvi_hwpool_s(void)
{
qvim_rc = qvi_construct_rc(cpus);
if (qvim_rc != QV_SUCCESS) return;

qvim_rc = qvi_hwloc_bitmap_calloc(&obcpuset);
}

~qvi_hwpool_s(void)
{
qvi_hwloc_bitmap_free(&obcpuset);
}
};

int
qvi_hwpool_new(
qvi_hwpool_t **rpool
) {
int rc = QV_SUCCESS;

qvi_hwpool_t *irpool = qvi_new qvi_hwpool_t();
if (!irpool) {
rc = QV_ERR_OOR;
goto out;
}

irpool->devinfos = qvi_new qvi_hwpool_devinfos_t();
if (!irpool->devinfos) {
rc = QV_ERR_OOR;
goto out;
}

rc = qvi_hwloc_bitmap_calloc(&irpool->cpuset);
if (rc != QV_SUCCESS) goto out;

rc = qvi_hwloc_bitmap_calloc(&irpool->obcpuset);
out:
int rc = qvi_new_rc(irpool);
if (rc != QV_SUCCESS) {
qvi_hwpool_free(&irpool);
}
Expand All @@ -97,7 +148,7 @@ qvi_hwpool_new_from_line(
int rc = qvi_hwpool_new(&irpool);
if (rc != QV_SUCCESS) goto out;

rc = qvi_hwloc_bitmap_copy(line->cpuset, irpool->cpuset);
rc = qvi_hwloc_bitmap_copy(line->cpuset, irpool->cpus.cpuset);
if (rc != QV_SUCCESS) goto out;

for (int i = 0; i < line->ndevinfos; ++i) {
Expand Down Expand Up @@ -125,15 +176,15 @@ qvi_hwpool_new_line_from_hwpool(
qvi_line_hwpool_t **line
) {
int rc = QV_SUCCESS;
const int ndevinfos = rpool->devinfos->size();
const int ndevinfos = rpool->devinfos.size();

qvi_line_hwpool_t *iline = nullptr;
rc = qvi_line_hwpool_new(&iline);
if (rc != QV_SUCCESS) goto out;
// Initialize and copy the cpuset.
rc = qvi_hwloc_bitmap_calloc(&iline->cpuset);
if (rc != QV_SUCCESS) goto out;
rc = qvi_hwloc_bitmap_copy(rpool->cpuset, iline->cpuset);
rc = qvi_hwloc_bitmap_copy(rpool->cpus.cpuset, iline->cpuset);
if (rc != QV_SUCCESS) goto out;
// Initialize and fill in the device information.
iline->ndevinfos = ndevinfos;
Expand All @@ -144,7 +195,7 @@ qvi_hwpool_new_line_from_hwpool(
}
do {
int idx = 0, nw = 0;
for (const auto &dinfo : *rpool->devinfos) {
for (const auto &dinfo : rpool->devinfos) {
iline->devinfos[idx].type = dinfo.second->type;
iline->devinfos[idx].id = dinfo.second->id;
// Initialize and copy cpuset
Expand Down Expand Up @@ -191,9 +242,6 @@ qvi_hwpool_free(
if (!rpool) return;
qvi_hwpool_t *irpool = *rpool;
if (!irpool) goto out;
delete irpool->devinfos;
qvi_hwloc_bitmap_free(&irpool->cpuset);
qvi_hwloc_bitmap_free(&irpool->obcpuset);
delete irpool;
out:
*rpool = nullptr;
Expand All @@ -204,7 +252,7 @@ qvi_hwpool_init(
qvi_hwpool_t *rpool,
hwloc_const_bitmap_t cpuset
) {
return qvi_hwloc_bitmap_copy(cpuset, rpool->cpuset);
return qvi_hwloc_bitmap_copy(cpuset, rpool->cpus.cpuset);
}

int
Expand All @@ -216,7 +264,7 @@ qvi_hwpool_add_device(
cstr_t uuid,
hwloc_const_cpuset_t affinity
) {
rpool->devinfos->insert(
rpool->devinfos.insert(
std::make_pair(
type,
std::make_shared<qvi_devinfo_t>(
Expand All @@ -231,24 +279,24 @@ int
qvi_hwpool_release_devices(
qvi_hwpool_t *pool
) {
pool->devinfos->clear();
pool->devinfos.clear();
return QV_SUCCESS;
}

hwloc_const_cpuset_t
qvi_hwpool_cpuset_get(
qvi_hwpool_t *rpool
) {
if (!rpool) return nullptr;
return rpool->cpuset;
assert(rpool);
return rpool->cpus.cpuset;
}

const qvi_hwpool_devinfos_t *
qvi_hwpool_devinfos_get(
qvi_hwpool_t *pool
) {
if (!pool) return nullptr;
return pool->devinfos;
assert(pool);
return &pool->devinfos;
}

#if 0
Expand Down Expand Up @@ -344,15 +392,15 @@ qvi_hwpool_add_devices_with_affinity(
const qv_hw_obj_type_t type = devts[i];
int nobjs = 0;
rc = qvi_hwloc_get_nobjs_in_cpuset(
hwloc, type, pool->cpuset, &nobjs
hwloc, type, pool->cpus.cpuset, &nobjs
);
if (rc != QV_SUCCESS) break;
// Iterate over the number of devices in each type.
for (int devi = 0; devi < nobjs; ++devi) {
char *devids = nullptr, *pcibid = nullptr, *uuids = nullptr;
// Device ID
rc = qvi_hwloc_get_device_in_cpuset(
hwloc, type, devi, pool->cpuset,
hwloc, type, devi, pool->cpus.cpuset,
QV_DEVICE_ID_ORDINAL, &devids
);
if (rc != QV_SUCCESS) break;
Expand All @@ -362,13 +410,13 @@ qvi_hwpool_add_devices_with_affinity(
if (rc != QV_SUCCESS) break;
// PCI Bus ID
rc = qvi_hwloc_get_device_in_cpuset(
hwloc, type, devi, pool->cpuset,
hwloc, type, devi, pool->cpus.cpuset,
QV_DEVICE_ID_PCI_BUS_ID, &pcibid
);
if (rc != QV_SUCCESS) break;
// UUID
rc = qvi_hwloc_get_device_in_cpuset(
hwloc, type, devi, pool->cpuset,
hwloc, type, devi, pool->cpus.cpuset,
QV_DEVICE_ID_UUID, &uuids
);
if (rc != QV_SUCCESS) break;
Expand Down
35 changes: 34 additions & 1 deletion src/qvi-utils.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2020-2022 Triad National Security, LLC
* Copyright (c) 2020-2024 Triad National Security, LLC
* All rights reserved.
*
* Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC
Expand All @@ -19,6 +19,39 @@

#include "qvi-common.h"

#ifdef __cplusplus

/**
* Returns the code captured by a constructor. Since we are trying to avoid the
* use of exceptions, we instead use this method for checking the state of an
* object after its construction. This method isn't perfect, and requires that a
* class member named qvim_rc (read as (qvi) (m)ember (r)eturn (c)ode) is
* present in the provided class. Compilation will fail otherwise.
*/
template <class T>
int
qvi_construct_rc(
const T &t
) {
return t.qvim_rc;
}

/**
* Similar to qvi_construct_rc(), but is used also to check the value returned
* by qvi_new.
*/
template <class T>
int
qvi_new_rc(
const T *const t
) {
// qvi_new must have returned nullptr.
if (!t) return QV_ERR_OOR;
return t->qvim_rc;
}

#endif

#ifdef __cplusplus
extern "C" {
#endif
Expand Down

0 comments on commit 2881434

Please sign in to comment.