diff --git a/sw/nic/gpuagent/api/gpu_api.cc b/sw/nic/gpuagent/api/gpu_api.cc index f2dda4d..f43c7a9 100644 --- a/sw/nic/gpuagent/api/gpu_api.cc +++ b/sw/nic/gpuagent/api/gpu_api.cc @@ -114,6 +114,10 @@ aga_gpu_info_from_entry (void *entry, void *ctxt) // some API operation is in progress on this object, skip it return false; } + if (gpu->is_parent_gpu()) { + // no need to show parent GPUs as they are placeholders + return false; + } memset(&info, 0, sizeof(aga_gpu_info_t)); // call entry read gpu->read(&info); diff --git a/sw/nic/gpuagent/api/include/aga_gpu.hpp b/sw/nic/gpuagent/api/include/aga_gpu.hpp index 187359b..9fa375b 100644 --- a/sw/nic/gpuagent/api/include/aga_gpu.hpp +++ b/sw/nic/gpuagent/api/include/aga_gpu.hpp @@ -752,6 +752,24 @@ typedef struct aga_gpu_info_s { aga_gpu_stats_t stats; } aga_gpu_info_t; +/// GPU profile +typedef struct aga_gpu_profile_s { + /// GPU id + uint32_t id; + /// GPU uuid + aga_obj_key_t key; + /// GPU handle + aga_gpu_handle_t handle; + /// is the GPU partition capable + bool partition_capable; + /// GPU compute partition type + aga_gpu_compute_partition_type_t compute_partition; + /// GPU compute partition id + uint32_t partition_id; + /// GPU memory partition type + aga_gpu_memory_partition_type_t memory_partition; +} aga_gpu_profile_t; + /// device type enum typedef enum aga_device_type_e { AGA_DEVICE_TYPE_NONE = 0, diff --git a/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc b/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc index d6fbb1a..de66728 100644 --- a/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc +++ b/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc @@ -745,59 +745,6 @@ smi_fill_vram_status_ (aga_gpu_handle_t gpu_handle, return SDK_RET_OK; } -sdk_ret_t -smi_get_gpu_partition_info (aga_gpu_handle_t gpu_handle, bool *capable, - aga_gpu_compute_partition_type_t *compute_partition, - aga_gpu_memory_partition_type_t *memory_partition) -{ - amdsmi_status_t amdsmi_ret; - amdsmi_gpu_metrics_t metrics_info = {}; - char partition_type[AGA_MAX_STR_LEN + 1]; - - *capable = true; - *compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_NONE; - *memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NONE; - // to deduce partition capability of platform, we rely on - // metrics field num_partition of a GPU field to be 0xffff - // on partition supported platform, this api is not supported - // for paritioned GPU other than index 0 or first_handle - // we mark the capablity to true on such cases to specify platform - // partition capability - amdsmi_ret = amdsmi_get_gpu_metrics_info(gpu_handle, - &metrics_info); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get GPU metrics info for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - if ((metrics_info.num_partition & 0xffff) == - AMDSMI_INVALID_PARTITION_COUNT) { - // this is unsupported platform like Mi2xx - *capable = false; - } - } - // fill compute partition type - amdsmi_ret = amdsmi_get_gpu_compute_partition(gpu_handle, - partition_type, AGA_MAX_STR_LEN + 1); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get compute partition for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - *compute_partition = - smi_to_aga_gpu_compute_partition_type(partition_type); - } - // fill memory partition type - amdsmi_ret = amdsmi_get_gpu_memory_partition(gpu_handle, - partition_type, AGA_MAX_STR_LEN + 1); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get memory partition for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - *memory_partition = - smi_to_aga_gpu_memory_partition_type(partition_type); - } - return SDK_RET_OK; -} - sdk_ret_t smi_get_gpu_partition_id (aga_gpu_handle_t gpu_handle, uint32_t *partition_id) { @@ -1611,6 +1558,39 @@ smi_gpu_fill_device_topology (aga_gpu_handle_t gpu_handle, return SDK_RET_OK; } +/// \brief function to get parent aga_obj_key_t for a given GPU +/// \param[in] gpu_handle GPU handle +/// \param[out] key aga_obj_key_t of the parent GPU +sdk_ret_t +smi_get_parent_gpu_uuid (aga_gpu_handle_t gpu_handle, aga_obj_key_t *parent_key) +{ + amdsmi_bdf_t pcie_bdf; + amdsmi_status_t status; + amdsmi_asic_info_t asic_info; + + // get device BDF + status = amdsmi_get_gpu_device_bdf(gpu_handle, &pcie_bdf); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get PCIe BDF of GPU {}, err {}", + gpu_handle, status); + return amdsmi_ret_to_sdk_ret(status); + } + // get GPU asic info + status = amdsmi_get_gpu_asic_info(gpu_handle, &asic_info); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get ASIC information of GPU {}, err {}", + gpu_handle, status); + return amdsmi_ret_to_sdk_ret(status); + } + // compute the parent GPU uuid + parent_key->reset(); + memcpy(&parent_key->id[0], asic_info.asic_serial, 4); + *(uint16_t *)(&parent_key->id[6]) = 0x1000; + *(uint64_t *)(&parent_key->id[8]) = pcie_bdf.as_uint; + + return SDK_RET_OK; +} + /// \brief function to get aga_obj_key_t for a given GPU /// \param[in] gpu_handle GPU handle /// \param[out] key aga_obj_key_t of the GPU @@ -1645,9 +1625,80 @@ smi_gpu_uuid_get (aga_gpu_handle_t gpu_handle, aga_obj_key_t *key) return SDK_RET_OK; } +/// \brief function to get profile of a given GPU +/// \param[in] id GPU id +/// \param[in] handle GPU handle +/// \param[out] gpu GPU profile +static sdk_ret_t +smi_fill_gpu_profile_ (uint32_t id, aga_gpu_handle_t handle, + aga_gpu_profile_t& gpu) +{ + sdk_ret_t ret; + amdsmi_status_t status; + char partition_type[AGA_MAX_STR_LEN + 1]; + amdsmi_accelerator_partition_profile_t profile; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PARTITIONS] = { 0 }; + + gpu.id = id; + gpu.handle = handle; + // get GPU uuid + ret = smi_gpu_uuid_get(gpu.handle, &gpu.key); + if (ret != SDK_RET_OK) { + AGA_TRACE_ERR("GPU discovery failed due to error in getting UUID of " + "GPU {}", gpu.handle); + return ret; + } + // get accelerator partition profile + status = amdsmi_get_gpu_accelerator_partition_profile(gpu.handle, &profile, + partition_id); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + if (status == AMDSMI_STATUS_NOT_SUPPORTED) { + // set to default values; this could be a partition as this API is + // not supported on partitions + gpu.compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_NONE; + gpu.partition_id = partition_id[0]; + gpu.partition_capable = true; + } else { + AGA_TRACE_ERR("GPU discovery failed due to error in getting " + "partition profile of GPU {}, err {}", gpu.key.str(), + status); + return amdsmi_ret_to_sdk_ret(status); + } + } else { + gpu.compute_partition = + smi_to_aga_gpu_compute_partition_type(profile.profile_type); + gpu.partition_id = partition_id[0]; + if (profile.num_partitions == std::numeric_limits::max()) { + // num partitions is set to the max uint32_t value for GPUs which + // don't support partitions + gpu.partition_capable = false; + } else { + gpu.partition_capable = true; + } + } + // set memory partition type + status = amdsmi_get_gpu_memory_partition(gpu.handle, partition_type, + AGA_MAX_STR_LEN + 1); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + if (status == AMDSMI_STATUS_NOT_SUPPORTED) { + // set to default values; this could be a partition as this API is + // not supported on partitions + gpu.memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NONE; + } else { + AGA_TRACE_ERR("GPU discovery failed due to error in getting " + "memory partition of GPU {}, err {}", gpu.key.str(), + status); + return amdsmi_ret_to_sdk_ret(status); + } + } else { + gpu.memory_partition = + smi_to_aga_gpu_memory_partition_type(partition_type); + } + return SDK_RET_OK; +} + sdk_ret_t -smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, - aga_obj_key_t *gpu_keys) +smi_discover_gpus (uint32_t *num_gpu, aga_gpu_profile_t *gpu) { sdk_ret_t ret; uint32_t num_procs; @@ -1657,10 +1708,10 @@ smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, amdsmi_socket_handle socket_handles[AGA_MAX_SOCKET]; aga_gpu_handle_t proc_handles[AGA_MAX_PROCESSORS_PER_SOCKET]; - if (!num_gpus) { + if (!num_gpu) { return SDK_RET_ERR; } - *num_gpus = 0; + *num_gpu = 0; // get the socket count available in the system status = amdsmi_get_socket_handles(&num_sockets, NULL); if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { @@ -1704,18 +1755,14 @@ smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, return amdsmi_ret_to_sdk_ret(status); } if (proc_type == AMDSMI_PROCESSOR_TYPE_AMD_GPU) { - gpu_handles[*num_gpus] = proc_handles[j]; - if (gpu_keys) { - ret = smi_gpu_uuid_get(proc_handles[j], - &gpu_keys[*num_gpus]); - if (ret != SDK_RET_OK) { - AGA_TRACE_ERR("GPU discovery failed due to error in " - "getting UUID of GPU {}", - proc_handles[j]); - return ret; - } + ret = smi_fill_gpu_profile_(*num_gpu, proc_handles[j], + gpu[*num_gpu]); + if (ret != SDK_RET_OK) { + AGA_TRACE_ERR("GPU discovery failed when processing GPU {}", + proc_handles[j]); + return ret; } - (*num_gpus)++; + (*num_gpu)++; } } } diff --git a/sw/nic/gpuagent/api/smi/amdsmi/smi_state.cc b/sw/nic/gpuagent/api/smi/amdsmi/smi_state.cc index fcfa53b..e878a0e 100644 --- a/sw/nic/gpuagent/api/smi/amdsmi/smi_state.cc +++ b/sw/nic/gpuagent/api/smi/amdsmi/smi_state.cc @@ -1508,6 +1508,7 @@ sdk_ret_t smi_state::init(aga_api_init_params_t *init_params) { sdk_ret_t ret; amdsmi_status_t status; + aga_gpu_profile_t gpu[AGA_MAX_GPU]; // initialize smi library status = amdsmi_init(AMDSMI_INIT_AMD_GPUS); @@ -1516,10 +1517,13 @@ smi_state::init(aga_api_init_params_t *init_params) { return amdsmi_ret_to_sdk_ret(status); } // discover gpus - ret = aga::smi_discover_gpus(&num_gpu_, gpu_handles_, NULL); + ret = aga::smi_discover_gpus(&num_gpu_, gpu); if (ret != SDK_RET_OK) { return ret; } + for (uint32_t i = 0; i < num_gpu_; i++) { + gpu_handles_[i] = gpu[i].handle; + } // spawn event monitor thread spawn_event_monitor_thread_(); // spawn watcher thread diff --git a/sw/nic/gpuagent/api/smi/amdsmi/smi_utils.hpp b/sw/nic/gpuagent/api/smi/amdsmi/smi_utils.hpp index 7bc6d53..5bf740c 100644 --- a/sw/nic/gpuagent/api/smi/amdsmi/smi_utils.hpp +++ b/sw/nic/gpuagent/api/smi/amdsmi/smi_utils.hpp @@ -357,24 +357,29 @@ smi_to_aga_gpu_page_status (amdsmi_memory_page_status_t page_status) return AGA_GPU_PAGE_STATUS_NONE; } -/// \brief convert amdsmi partition type string to aga comptue partition type +/// \brief convert amdsmi accelerator partition to aga comptue partition type /// \param[in] partition_type amdsmi partition type string /// \return aga gpu compute partition type static inline aga_gpu_compute_partition_type_t -smi_to_aga_gpu_compute_partition_type (std::string partition_type) +smi_to_aga_gpu_compute_partition_type ( + amdsmi_accelerator_partition_type_t partition_type) { - if (partition_type == "SPX") { + switch (partition_type) { + case AMDSMI_ACCELERATOR_PARTITION_SPX: return AGA_GPU_COMPUTE_PARTITION_TYPE_SPX; - } else if (partition_type == "DPX") { + case AMDSMI_ACCELERATOR_PARTITION_DPX: return AGA_GPU_COMPUTE_PARTITION_TYPE_DPX; - } else if (partition_type == "TPX") { + case AMDSMI_ACCELERATOR_PARTITION_TPX: return AGA_GPU_COMPUTE_PARTITION_TYPE_TPX; - } else if (partition_type == "QPX") { + case AMDSMI_ACCELERATOR_PARTITION_QPX: return AGA_GPU_COMPUTE_PARTITION_TYPE_QPX; - } else if (partition_type == "CPX") { + case AMDSMI_ACCELERATOR_PARTITION_CPX: return AGA_GPU_COMPUTE_PARTITION_TYPE_CPX; + default: + break; } + return AGA_GPU_COMPUTE_PARTITION_TYPE_NONE; } diff --git a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc index 76984ce..43ad006 100644 --- a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc +++ b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc @@ -597,43 +597,6 @@ smi_fill_vram_status_ (aga_gpu_handle_t gpu_handle, return SDK_RET_OK; } -sdk_ret_t -smi_get_gpu_partition_info (aga_gpu_handle_t gpu_handle, bool *capable, - aga_gpu_compute_partition_type_t *compute_partition, - aga_gpu_memory_partition_type_t *memory_partition) -{ - amdsmi_status_t amdsmi_ret; - amdsmi_memory_partition_config_t mem_part_config = {}; - amdsmi_accelerator_partition_profile_t part_config = {}; - uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PARTITIONS]; - - // set as false for now - *capable = false; - *compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_NONE; - *memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NONE; - // fill compute partition type - amdsmi_ret = amdsmi_get_gpu_accelerator_partition_profile(gpu_handle, - &part_config, partition_id); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get accelerator partition config for GPU {}, " - "err {}", gpu_handle, amdsmi_ret); - } else { - *compute_partition = - smi_to_aga_gpu_compute_partition_type(part_config.profile_type); - } - // fill memory partition type - amdsmi_ret = amdsmi_get_gpu_memory_partition_config(gpu_handle, - &mem_part_config); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get memory partition config for GPU {}, " - "err {}", gpu_handle, amdsmi_ret); - } else { - *memory_partition = - smi_to_aga_gpu_memory_partition_type(mem_part_config.mp_mode); - } - return SDK_RET_OK; -} - sdk_ret_t smi_get_gpu_partition_id (aga_gpu_handle_t gpu_handle, uint32_t *partition_id) { @@ -983,6 +946,39 @@ smi_gpu_fill_device_topology (aga_gpu_handle_t gpu_handle, return SDK_RET_OK; } +/// \brief function to get parent aga_obj_key_t for a given GPU +/// \param[in] gpu_handle GPU handle +/// \param[out] key aga_obj_key_t of the parent GPU +sdk_ret_t +smi_get_parent_gpu_uuid (aga_gpu_handle_t gpu_handle, aga_obj_key_t *parent_key) +{ + amdsmi_bdf_t pcie_bdf; + amdsmi_status_t status; + amdsmi_asic_info_t asic_info; + + // get device BDF + status = amdsmi_get_gpu_device_bdf(gpu_handle, &pcie_bdf); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get PCIe BDF of GPU {}, err {}", + gpu_handle, status); + return amdsmi_ret_to_sdk_ret(status); + } + // get GPU asic info + status = amdsmi_get_gpu_asic_info(gpu_handle, &asic_info); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get ASIC information of GPU {}, err {}", + gpu_handle, status); + return amdsmi_ret_to_sdk_ret(status); + } + // compute the parent GPU uuid + parent_key->reset(); + memcpy(&parent_key->id[0], asic_info.asic_serial, 4); + *(uint16_t *)(&parent_key->id[6]) = 0x1000; + *(uint64_t *)(&parent_key->id[8]) = pcie_bdf.as_uint; + + return SDK_RET_OK; +} + /// \brief function to get aga_obj_key_t for a given GPU /// \param[in] gpu_handle GPU handle /// \param[out] key aga_obj_key_t of the GPU @@ -1017,19 +1013,73 @@ smi_gpu_uuid_get (aga_gpu_handle_t gpu_handle, aga_obj_key_t *key) return SDK_RET_OK; } +static sdk_ret_t +smi_fill_gpu_profile_ (uint32_t id, aga_gpu_handle_t handle, + aga_gpu_profile_t& gpu) +{ + sdk_ret_t ret; + amdsmi_status_t status; + amdsmi_accelerator_partition_profile_t profile; + amdsmi_memory_partition_config_t mem_part_config = {}; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PARTITIONS] = { 0 }; + + gpu.id = id; + gpu.handle = handle; + // get GPU uuid + ret = smi_gpu_uuid_get(gpu.handle, &gpu.key); + if (ret != SDK_RET_OK) { + AGA_TRACE_ERR("GPU discovery failed due to error in getting UUID of " + "GPU {}", gpu.handle); + return ret; + } + // set partition information to default values + gpu.partition_capable = false; + gpu.partition_id = 0; + gpu.compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_NONE; + gpu.memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NONE; + // get accelerator partition profile + status = amdsmi_get_gpu_accelerator_partition_profile(gpu.handle, &profile, + partition_id); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get accelerator partition config for GPU {}, " + "err {}", gpu.key.str(), status); + } else { + gpu.compute_partition = + smi_to_aga_gpu_compute_partition_type(profile.profile_type); + gpu.partition_id = partition_id[0]; + if (profile.num_partitions == std::numeric_limits::max()) { + // num partitions is set to the max uint32_t value for GPUs which + // don't support partitions + gpu.partition_capable = false; + } else { + gpu.partition_capable = true; + } + } + // set memory partition type + status = amdsmi_get_gpu_memory_partition_config(gpu.handle, + &mem_part_config); + if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get memory partition config for GPU {}, " + "err {}", gpu.key.str(), status); + } else { + gpu.memory_partition = + smi_to_aga_gpu_memory_partition_type(mem_part_config.mp_mode); + } + return SDK_RET_OK; +} + sdk_ret_t -smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, - aga_obj_key_t *gpu_keys) +smi_discover_gpus (uint32_t *num_gpu, aga_gpu_profile_t *gpu) { sdk_ret_t ret; uint32_t num_procs; amdsmi_status_t status; aga_gpu_handle_t proc_handles[AGA_MAX_GPU]; - if (!num_gpus) { + if (!num_gpu) { return SDK_RET_ERR; } - *num_gpus = 0; + *num_gpu = 0; // for each socket get the number of processors status = amdsmi_get_processor_handles(NULL, &num_procs, NULL); if (unlikely(status != AMDSMI_STATUS_SUCCESS)) { @@ -1045,18 +1095,14 @@ smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, } // get uuids of each GPU for (uint32_t j = 0; j < num_procs; j++) { - gpu_handles[*num_gpus] = proc_handles[j]; - if (gpu_keys) { - ret = smi_gpu_uuid_get(proc_handles[j], - &gpu_keys[*num_gpus]); - if (ret != SDK_RET_OK) { - AGA_TRACE_ERR("GPU discovery failed due to error in " - "getting UUID of GPU {}", - proc_handles[j]); - return ret; - } + ret = smi_fill_gpu_profile_(*num_gpu, proc_handles[j], + gpu[*num_gpu]); + if (ret != SDK_RET_OK) { + AGA_TRACE_ERR("GPU discovery failed when processing GPU {}", + proc_handles[j]); + return ret; } - (*num_gpus)++; + (*num_gpu)++; } return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_state.cc b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_state.cc index c1bcb40..514e314 100644 --- a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_state.cc +++ b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_state.cc @@ -695,6 +695,7 @@ sdk_ret_t smi_state::init(aga_api_init_params_t *init_params) { sdk_ret_t ret; amdsmi_status_t status; + aga_gpu_profile_t gpu[AGA_MAX_GPU]; // initialize smi library status = amdsmi_init(AMDSMI_INIT_AMD_GPUS); @@ -703,10 +704,13 @@ smi_state::init(aga_api_init_params_t *init_params) { return amdsmi_ret_to_sdk_ret(status); } // discover gpus - ret = aga::smi_discover_gpus(&num_gpu_, gpu_handles_, NULL); + ret = aga::smi_discover_gpus(&num_gpu_, gpu); if (ret != SDK_RET_OK) { return ret; } + for (uint32_t i = 0; i < num_gpu_; i++) { + gpu_handles_[i] = gpu[i].handle; + } // spawn event monitor thread spawn_event_monitor_thread_(); // spawn watcher thread diff --git a/sw/nic/gpuagent/api/smi/smi_api.hpp b/sw/nic/gpuagent/api/smi/smi_api.hpp index 9097bef..d603374 100644 --- a/sw/nic/gpuagent/api/smi/smi_api.hpp +++ b/sw/nic/gpuagent/api/smi/smi_api.hpp @@ -114,12 +114,15 @@ sdk_ret_t smi_gpu_fill_device_topology(aga_gpu_handle_t handle, /// \brief discover gpu devices /// \param[out] num_gpus number of GPU devices discovered -/// \param[out] gpu_handles handle of gpu devices discovered -/// \param[out] gpu_keys uuids of gpu devices discovered +/// \param[out] gpu profile of GPU devices discovered /// \return SDK_RET_OK or error code in case of failure -sdk_ret_t smi_discover_gpus(uint32_t *num_gpus, - aga_gpu_handle_t *gpu_handles, - aga_obj_key_t *gpu_keys); +sdk_ret_t smi_discover_gpus(uint32_t *num_gpu, aga_gpu_profile_t *gpu); + +/// \brief function to compute parent GPU's uuid +/// \param[in] gpu_handle handle of GPU device +/// \param[out] parent_key computed uuid of parent GPU +sdk_ret_t smi_get_parent_gpu_uuid(aga_gpu_handle_t gpu_handle, + aga_obj_key_t *parent_key); /// \brief function to get number of bad pages for GPU /// \param[in] gpu GPU object @@ -137,23 +140,6 @@ sdk_ret_t smi_gpu_get_bad_page_records(void *gpu_obj, uint32_t num_bad_pages, aga_gpu_bad_page_record_t *records); -/// \brief function to get GPU partition id -/// \param[in] gpu_handle handle of GPU device -/// \param[out] partition_id partition id of the GPU -sdk_ret_t smi_get_gpu_partition_id(aga_gpu_handle_t gpu_handle, - uint32_t *partition_id); - -/// \brief function to get GPU partition information -/// \param[in] gpu_handle handle of GPU device -/// \param[out] partition_capable true if platform supports partitioning -/// \param[out] compute_partition compute partition type -/// \param[out] memory_partition memory partition type -/// \return SDK_RET_OK or error code in case of failure -sdk_ret_t smi_get_gpu_partition_info(aga_gpu_handle_t gpu_handle, - bool *partition_capable, - aga_gpu_compute_partition_type_t *compute_partition, - aga_gpu_memory_partition_type_t *memory_partition); - /// \brief function to fill immutable attributes in GPU spec and status /// \param[in] gpu_handle handle of GPU device /// \param[out] spec GPU spec diff --git a/sw/nic/gpuagent/api/smi/smi_api_mock.cc b/sw/nic/gpuagent/api/smi/smi_api_mock.cc index 2b7204f..775ec37 100644 --- a/sw/nic/gpuagent/api/smi/smi_api_mock.cc +++ b/sw/nic/gpuagent/api/smi/smi_api_mock.cc @@ -643,46 +643,28 @@ smi_gpu_fill_device_topology (aga_gpu_handle_t gpu_handle, } sdk_ret_t -smi_get_gpu_partition_id (aga_gpu_handle_t gpu_handle, uint32_t *partition_id) +smi_get_parent_gpu_uuid (aga_gpu_handle_t gpu_handle, aga_obj_key_t *parent_key) { - *partition_id = 0; + *parent_key = g_gpu_map[gpu_handle].key; return SDK_RET_OK; } sdk_ret_t -smi_get_gpu_virtualization_mode (aga_gpu_handle_t gpu_handle, - aga_gpu_virtualization_mode_t *mode) -{ - *mode = AGA_VIRTUALIZATION_MODE_BAREMETAL; - return SDK_RET_OK; -} - -sdk_ret_t -smi_get_gpu_partition_info (aga_gpu_handle_t gpu_handle, bool *capable, - aga_gpu_compute_partition_type_t *compute_partition, - aga_gpu_memory_partition_type_t *memory_partition) -{ - *capable = true; - *compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_SPX; - *memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NPS1; - return SDK_RET_OK; -} - -sdk_ret_t -smi_discover_gpus (uint32_t *num_gpus, aga_gpu_handle_t *gpu_handles, - aga_obj_key_t *gpu_keys) +smi_discover_gpus (uint32_t *num_gpus, aga_gpu_profile_t *gpu) { if (!num_gpus) { return SDK_RET_ERR; } - *num_gpus = AGA_MOCK_NUM_GPU; + *num_gpu = AGA_MOCK_NUM_GPU; for (uint32_t i = 0; i < *num_gpus; i++) { - gpu_handles[i] = gpu_get_handle(i); - } - if (gpu_keys) { - for (uint32_t i = 0; i < *num_gpus; i++) { - gpu_keys[i] = gpu_uuid(i, gpu_get_unique_id(i)); - } + gpu[i].handle = gpu_get_handle(i); + gpu[i].key = gpu_uuid(i, gpu_get_unique_id(i)); + // set GPU ids + gpu[i].id = i; + // set partition information + gpu[i].partition_capable = true; + gpu[i].compute_partition = AGA_GPU_COMPUTE_PARTITION_TYPE_SPX; + gpu[i].memory_partition = AGA_GPU_MEMORY_PARTITION_TYPE_NPS1; } return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/cli/cmd/gpu.go b/sw/nic/gpuagent/cli/cmd/gpu.go index 9644c83..fbb0054 100644 --- a/sw/nic/gpuagent/cli/cmd/gpu.go +++ b/sw/nic/gpuagent/cli/cmd/gpu.go @@ -139,8 +139,6 @@ func init() { gpuShowCmd.Flags().BoolP("status", "s", false, "Show GPU status") gpuShowCmd.Flags().Bool("summary", false, "Display number of objects") gpuShowCmd.Flags().StringVarP(&gpuID, "id", "i", "", "Specify GPU id") - gpuShowCmd.Flags().BoolP("partitioned", "p", false, - "Show only partitioned GPUs") gpuShowCmd.AddCommand(gpuAllShowCmd) gpuAllShowCmd.Flags().StringVarP(&gpuID, "id", "i", "", "Specify GPU id") @@ -217,7 +215,7 @@ func printGPUPartitions(resp *aga.GPUComputePartition) { if i != 0 { fmt.Printf("%-56s%-40s\n", "", utils.IdToStr(partition)) } else { - fmt.Printf("%-40s\n", "", utils.IdToStr(partition)) + fmt.Printf("%-40s\n", utils.IdToStr(partition)) } } } @@ -502,23 +500,9 @@ func gpuShowCmdHandler(cmd *cobra.Command, args []string) error { return fmt.Errorf("Operation failed with %v error", respMsg.ApiStatus) } - var response []*aga.GPU - for _, resp := range respMsg.Response { - status := resp.GetStatus() - if len(status.GetGPUPartition()) > 0 { - if cmd != nil && cmd.Flags().Changed("partitioned") { - response = append(response, resp) - } - } else { - if cmd == nil || !cmd.Flags().Changed("partitioned") { - response = append(response, resp) - } - } - } - // print GPUs if cmd != nil && cmd.Flags().Changed("yaml") { - yamlArr, _ := yaml.Marshal(response) + yamlArr, _ := yaml.Marshal(respMsg.Response) fmt.Println(string(yamlArr)) } else if cmd != nil && cmd.Flags().Changed("json") { // json output requires that all GPUs are listed within [] braces @@ -526,7 +510,7 @@ func gpuShowCmdHandler(cmd *cobra.Command, args []string) error { fmt.Printf("[\n") } rcvdResp := false - for _, resp := range response { + for _, resp := range respMsg.Response { if rcvdResp == true { // json output requires a , after each GPU fmt.Printf(",\n") @@ -539,17 +523,17 @@ func gpuShowCmdHandler(cmd *cobra.Command, args []string) error { fmt.Printf("\n]\n") } } else if cmd != nil && cmd.Flags().Changed("summary") { - printGPUSummary(len(response)) + printGPUSummary(len(respMsg.Response)) } else if cmd != nil && cmd.Flags().Changed("status") { - for _, resp := range response { + for _, resp := range respMsg.Response { printGPUStatus(resp, true) } - printGPUSummary(len(response)) + printGPUSummary(len(respMsg.Response)) } else { - for _, resp := range response { + for _, resp := range respMsg.Response { printGPUSpec(resp, true) } - printGPUSummary(len(response)) + printGPUSummary(len(respMsg.Response)) } return nil } @@ -804,7 +788,7 @@ func printGPUSpec(gpu *aga.GPU, specOnly bool) { } // TODO: fill GPU RAS Spec if specOnly { - fmt.Printf("\n%s\n", strings.Repeat("-", 80)) + fmt.Printf("\n%s\n", strings.Repeat("-", 90)) } } @@ -1059,7 +1043,7 @@ func printGPUStatus(gpu *aga.GPU, statusOnly bool) { } } if statusOnly { - fmt.Printf("\n%s\n", strings.Repeat("-", 80)) + fmt.Printf("\n%s\n", strings.Repeat("-", 90)) } } @@ -1858,7 +1842,7 @@ func printGPUStats(gpu *aga.GPU, statsOnly bool) { } } - fmt.Printf("\n%s\n", strings.Repeat("-", 80)) + fmt.Printf("\n%s\n", strings.Repeat("-", 90)) } type ShadowGPU struct { diff --git a/sw/nic/gpuagent/init.cc b/sw/nic/gpuagent/init.cc index 8e4edff..8579dad 100644 --- a/sw/nic/gpuagent/init.cc +++ b/sw/nic/gpuagent/init.cc @@ -56,91 +56,86 @@ create_gpus (void) { sdk_ret_t ret; uint32_t num_gpu; - uint32_t partition_id; - bool partition_capable; aga_gpu_spec_t spec = { 0 }; - aga_obj_key_t gpu_key[AGA_MAX_GPU]; - aga_gpu_handle_t gpu_handles[AGA_MAX_GPU]; - std::unordered_map key_count_map; - std::unordered_map parent_gpu_map; + aga::gpu_entry *parent_gpu = NULL; + aga_gpu_profile_t gpu[AGA_MAX_GPU]; - ret = aga::smi_discover_gpus(&num_gpu, gpu_handles, gpu_key); + ret = aga::smi_discover_gpus(&num_gpu, gpu); if (ret != SDK_RET_OK) { AGA_TRACE_ERR("GPU discovery failed, err {}", ret()); return SDK_RET_ERR; } - // go through GPUs to detect partitions if any; when a GPU is - // partitioned, all the children share the same UUID; we use this to detect - // partitions - for (uint32_t i = 0; i < num_gpu; i++) { - key_count_map[gpu_key[i]]++; - } - // first create parent GPUs for any partitioned GPUs discovered + // NOTE: + // when a GPU is partitioned, the partitions are in sequential order + + // start creating the GPU objects + AGA_TRACE_DEBUG("Creating {} GPU objects ...", num_gpu); for (uint32_t i = 0; i < num_gpu; i++) { - // check if it is a partitioned GPU (multiple GPUs with the same UUID) - if (key_count_map[gpu_key[i]] > 1) { - if (parent_gpu_map.find(gpu_key[i]) != parent_gpu_map.end()) { - // parent GPU already created + // create parent GPUs for partitioned GPUs; we check if it is a + // partitioned GPU by checking the compute partition type is not NONE or + // SPX; this is only set for the first partition (partition id 0) + if ((gpu[i].compute_partition != AGA_GPU_COMPUTE_PARTITION_TYPE_NONE) && + (gpu[i].compute_partition != AGA_GPU_COMPUTE_PARTITION_TYPE_SPX)) { + // this is the first partition; create the parent GPU + // construct parent GPU uuid + ret = aga::smi_get_parent_gpu_uuid(gpu[i].handle, &spec.key); + if (ret != SDK_RET_OK) { + AGA_TRACE_ERR("Failed to compute parent GPU uuid for GPU {}", + gpu[i].key.str()); + // continue to next gpu continue; } - // set parent GPU uuid - spec.key = gpu_key[i]; // parent GPUs cannot have a parent themselves spec.parent_gpu.reset(); - } else { - continue; - } - AGA_TRACE_DEBUG("Creating parent GPU {}", spec.key.str()); - // attempt to create gpu object - ret = aga_gpu_create(&spec); - if (unlikely(ret != SDK_RET_OK)) { - AGA_TRACE_ERR("GPU {} creation failed, err {}", spec.key.str(), - ret()); - // continue to next gpu - continue; - } - // add parent GPU to map - parent_gpu_map[spec.key] = true; - } - // start creating the GPU objects - AGA_TRACE_DEBUG("Creating {} GPU objects ...", num_gpu); - for (uint32_t i = 0; i < num_gpu; i++) { - spec.key = gpu_key[i]; - // check if it is a partitioned GPU - if (key_count_map[gpu_key[i]] > 1) { - // set parent GPU key - spec.parent_gpu = spec.key; - // set GPU partition ID - ret = aga::smi_get_gpu_partition_id(gpu_handles[i], - &partition_id); + // set partition types + spec.memory_partition_type = gpu[i].memory_partition; + spec.compute_partition_type = gpu[i].compute_partition; + AGA_TRACE_DEBUG("Creating parent GPU {}", spec.key.str()); + // attempt to create gpu object + ret = aga_gpu_create(&spec); if (unlikely(ret != SDK_RET_OK)) { - AGA_TRACE_ERR("GPU {} creation failed, err {}", spec.key.str(), - ret()); + AGA_TRACE_ERR("Parent GPU {} creation failed, err {}", + spec.key.str(), ret()); // continue to next gpu continue; } - // all child GPUs share the UUID of the parent; to differentiate we - // encode the partition id in the UUID - *(uint32_t *)&spec.key.id[4] = partition_id; + // find the parent GPU, so thht we can add children to it + parent_gpu = gpu_db()->find(&spec.key); + if (!parent_gpu) { + AGA_TRACE_ERR("Parent GPU {} entry not found", spec.key.str()); + // continue to next gpu + continue; + } + // stash first partition handle in the parent so that it can be + // propogated to all its children + parent_gpu->set_first_partition_handle(gpu[i].handle); + } else if (gpu[i].partition_id) { + // this is a subsequent partition; ignore if we didn't create a + // parent GPU + if (!parent_gpu) { + AGA_TRACE_ERR("Parent GPU not found for GPU {}, ignore ...", + gpu[i].key.str()); + continue; + } } else { - // set parent gpu to be invalid - spec.parent_gpu.reset(); - // set partition ID to invalid - partition_id = AGA_GPU_INVALID_PARTITION_ID; + // non-partition case; reset parent GPU + parent_gpu = NULL; } - // get partition informatino for GPU - ret = aga::smi_get_gpu_partition_info(gpu_handles[i], - &partition_capable, - &spec.compute_partition_type, - &spec.memory_partition_type); - if (unlikely(ret != SDK_RET_OK)) { - AGA_TRACE_ERR("Failed to get GPU {} partition information, err {}", - spec.key.str(), ret()); + // create GPU + spec.key = gpu[i].key; + if (parent_gpu) { + spec.parent_gpu = parent_gpu->key(); + // get the partition types from the parent GPUs + spec.memory_partition_type = parent_gpu->memory_partition_type(); + spec.compute_partition_type = parent_gpu->compute_partition_type(); + } else { + spec.parent_gpu.reset(); + spec.memory_partition_type = gpu[i].memory_partition; + spec.compute_partition_type = gpu[i].compute_partition; } - AGA_TRACE_DEBUG("Creating GPU {} id {} handle {}", - spec.key.str(), i, gpu_handles[i]); + AGA_TRACE_DEBUG("Creating GPU {} id {} handle {} partition id {}", + spec.key.str(), gpu[i].id, gpu[i].handle, + gpu[i].partition_id); // attempt to create gpu object ret = aga_gpu_create(&spec); if (unlikely(ret != SDK_RET_OK)) { @@ -157,64 +152,29 @@ create_gpus (void) // continue to next gpu continue; } - AGA_TRACE_DEBUG("GPU {} partition capability set {}", spec.key.str(), - entry->is_partionable()); // set GPU id - entry->set_id(i); + entry->set_id(gpu[i].id); // set GPU handle - entry->set_handle(gpu_handles[i]); + entry->set_handle(gpu[i].handle); // set partition capability - entry->set_partition_capability(partition_capable); + entry->set_partition_capability(gpu[i].partition_capable); // set partition id - entry->set_partition_id(partition_id); + entry->set_partition_id(gpu[i].partition_id); // initialize immutable attributes in GPU spec and status entry->init_immutable_attrs(); // insert in handle db gpu_db()->insert_in_handle_db(entry); // if GPU is a child GPU, add to the parent GPU if (spec.parent_gpu.valid()) { - auto parent_entry = gpu_db()->find(&spec.parent_gpu); - if (!parent_entry) { - // should we error out; this shouldn't happen - AGA_TRACE_ERR("Parent GPU {} for GPU {} not found", - spec.parent_gpu.str(), spec.key.str()); - continue; - } - parent_entry->add_child_gpu(&spec.key); - parent_entry->set_compute_partition_type( - spec.compute_partition_type); - parent_entry->set_memory_partition_type( - spec.memory_partition_type); - // if this child GPU is the first partition, stash its handle in the - // parent so that it can be propogated to all its children - if (!partition_id) { - parent_entry->set_first_partition_handle(gpu_handles[i]); - } + parent_gpu->add_child_gpu(&spec.key); + // set first partition handle + entry->set_first_partition_handle( + parent_gpu->first_partition_handle()); } else { // when the GPU is not partitioned set the first partition handle to // be the same as the GPU handle - entry->set_first_partition_handle(gpu_handles[i]); - } - } - for (auto& pair : parent_gpu_map) { - auto parent_gpu = gpu_db()->find((aga_obj_key_t *)&pair.first); - if (!parent_gpu) { - // shouldn't happen - AGA_TRACE_ERR("Parent GPU {} not found", pair.first.str()); - continue; + entry->set_first_partition_handle(gpu[i].handle); } - auto child_gpus = parent_gpu->child_gpus(); - for (uint32_t i = 0; i < child_gpus.size(); i++) { - auto child = gpu_db()->find(&child_gpus[i]); - if (!child) { - // shouldn't happen - AGA_TRACE_ERR("Child GPU {} not found", child_gpus[i].str()); - continue; - } - child->set_first_partition_handle( - parent_gpu->first_partition_handle()); - } - } return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/protos/gpu.proto b/sw/nic/gpuagent/protos/gpu.proto index a5d9e26..7aee6ae 100644 --- a/sw/nic/gpuagent/protos/gpu.proto +++ b/sw/nic/gpuagent/protos/gpu.proto @@ -416,11 +416,13 @@ message GPUStatus { uint32 PartitionId = 22; // GPU partitions (aka. child GPUs) // NOTE: - // only valid for physical GPUs which have been partitioned + // deprecated; no longer in use repeated bytes GPUPartition = 23; // physical GPU (aka. parent GPU) // NOTE: // only valid for GPU partitions (child GPUs) + // to get information about the physical GPU and its child GPUs use + // GPUComputePartitionGet bytes PhysicalGPU = 24; // GPU KFD id uint64 KFDId = 25;