[Pal/LibOS] Add physical id and fix cpu cores in /proc/cpuinfo

LibOS/shim/src/fs/proc/info.c

@@ -143,7 +143,7 @@ static int proc_cpuinfo_open(struct shim_handle* hdl, const char* name, int flag
         len += ret;                                                  \
     } while (0)
 
-    for (size_t n = 0; n < pal_control.cpu_info.cpu_num; n++) {
+    for (size_t n = 0; n < pal_control.cpu_info.online_logical_cores; n++) {
         /* Below strings must match exactly the strings retrieved from /proc/cpuinfo
          * (see Linux's arch/x86/kernel/cpu/proc.c) */
         ADD_INFO("processor\t: %lu\n", n);
@@ -152,8 +152,9 @@ static int proc_cpuinfo_open(struct shim_handle* hdl, const char* name, int flag
         ADD_INFO("model\t\t: %lu\n", pal_control.cpu_info.cpu_model);
         ADD_INFO("model name\t: %s\n", pal_control.cpu_info.cpu_brand);
         ADD_INFO("stepping\t: %lu\n", pal_control.cpu_info.cpu_stepping);
+        ADD_INFO("physical id\t: %d\n", pal_control.cpu_info.cpu_socket[n]);
         ADD_INFO("core id\t\t: %lu\n", n);
-        ADD_INFO("cpu cores\t: %lu\n", pal_control.cpu_info.cpu_num);
+        ADD_INFO("cpu cores\t: %lu\n", pal_control.cpu_info.physical_cores_per_socket);
         double bogomips = pal_control.cpu_info.cpu_bogomips;
         // Apparently graphene snprintf cannot into floats.
         ADD_INFO("bogomips\t: %lu.%02lu\n", (unsigned long)bogomips,

LibOS/shim/src/sys/shim_sched.c

@@ -160,7 +160,7 @@ static int check_affinity_params(int ncpus, size_t len, __kernel_cpu_set_t* user
 /* dummy implementation: ignore user-supplied mask and return success */
 int shim_do_sched_setaffinity(pid_t pid, size_t len, __kernel_cpu_set_t* user_mask_ptr) {
     __UNUSED(pid);
-    int ncpus = PAL_CB(cpu_info.cpu_num);
+    int ncpus = PAL_CB(cpu_info.online_logical_cores);
 
     int bitmask_size_in_bytes = check_affinity_params(ncpus, len, user_mask_ptr);
     if (bitmask_size_in_bytes < 0)
@@ -172,7 +172,7 @@ int shim_do_sched_setaffinity(pid_t pid, size_t len, __kernel_cpu_set_t* user_ma
 /* dummy implementation: always return all-ones (as many as there are host CPUs)  */
 int shim_do_sched_getaffinity(pid_t pid, size_t len, __kernel_cpu_set_t* user_mask_ptr) {
     __UNUSED(pid);
-    int ncpus = PAL_CB(cpu_info.cpu_num);
+    int ncpus = PAL_CB(cpu_info.online_logical_cores);
 
     int bitmask_size_in_bytes = check_affinity_params(ncpus, len, user_mask_ptr);
     if (bitmask_size_in_bytes < 0)

Pal/include/arch/x86_64/pal-arch.h

@@ -187,7 +187,12 @@ static inline bool pal_context_has_user_pagefault(PAL_CONTEXT* context) {
 
 /* PAL_CPU_INFO holds /proc/cpuinfo data */
 typedef struct PAL_CPU_INFO_ {
-    PAL_NUM cpu_num;
+    /* Number of logical cores available in the host */
+    PAL_NUM online_logical_cores;
+    /* Number of physical cores in a socket (physical package) */
+    PAL_NUM physical_cores_per_socket;
+    /* array of "logical core -> socket" mappings; has online_logical_cores elements */
+    int* cpu_socket;
     PAL_STR cpu_vendor;
     PAL_STR cpu_brand;
     PAL_NUM cpu_family;

Pal/regression/Bootstrap.c

@@ -62,7 +62,7 @@ int main(int argc, char** argv, char** envp) {
         (void*)&test_func < pal_control.executable_range.end)
         pal_printf("Executable Range OK\n");
 
-    pal_printf("CPU num: %ld\n", pal_control.cpu_info.cpu_num);
+    pal_printf("CPU num: %ld\n", pal_control.cpu_info.online_logical_cores);
     pal_printf("CPU vendor: %s\n", pal_control.cpu_info.cpu_vendor);
     pal_printf("CPU brand: %s\n", pal_control.cpu_info.cpu_brand);
     pal_printf("CPU family: %ld\n", pal_control.cpu_info.cpu_family);

Pal/src/host/Linux-SGX/db_main-x86_64.c

@@ -120,21 +120,27 @@ int _DkGetCPUInfo(PAL_CPU_INFO* ci) {
 
     const size_t VENDOR_ID_SIZE = 13;
     char* vendor_id = malloc(VENDOR_ID_SIZE);
-    _DkCpuIdRetrieve(0, 0, words);
+    if (!vendor_id)
+        return -PAL_ERROR_NOMEM;
 
+    _DkCpuIdRetrieve(0, 0, words);
     FOUR_CHARS_VALUE(&vendor_id[0], words[PAL_CPUID_WORD_EBX]);
     FOUR_CHARS_VALUE(&vendor_id[4], words[PAL_CPUID_WORD_EDX]);
     FOUR_CHARS_VALUE(&vendor_id[8], words[PAL_CPUID_WORD_ECX]);
     vendor_id[VENDOR_ID_SIZE - 1] = '\0';
     ci->cpu_vendor = vendor_id;
     // Must be an Intel CPU
     if (memcmp(vendor_id, "GenuineIntel", 12)) {
-        free(vendor_id);
-        return -PAL_ERROR_INVAL;
+        rv = -PAL_ERROR_INVAL;
+        goto out_vendor_id;
     }
 
     const size_t BRAND_SIZE = 49;
     char* brand = malloc(BRAND_SIZE);
+    if (!brand) {
+        rv = -PAL_ERROR_NOMEM;
+        goto out_vendor_id;
+    }
     _DkCpuIdRetrieve(0x80000002, 0, words);
     memcpy(&brand[ 0], words, sizeof(unsigned int) * PAL_CPUID_WORD_NUM);
     _DkCpuIdRetrieve(0x80000003, 0, words);
@@ -144,9 +150,9 @@ int _DkGetCPUInfo(PAL_CPU_INFO* ci) {
     brand[BRAND_SIZE - 1] = '\0';
     ci->cpu_brand = brand;
 
-    /* we cannot use CPUID(0xb) because it counts even disabled-by-BIOS cores (e.g. HT cores);
-     * instead, this is passed in via g_pal_sec at start-up time. */
-    ci->cpu_num = g_pal_sec.num_cpus;
+    ci->online_logical_cores = g_pal_sec.online_logical_cores;
+    ci->physical_cores_per_socket = g_pal_sec.physical_cores_per_socket;
+    ci->cpu_socket = g_pal_sec.cpu_socket;
 
     _DkCpuIdRetrieve(1, 0, words);
     ci->cpu_family   = BIT_EXTRACT_LE(words[PAL_CPUID_WORD_EAX],  8, 12) +
@@ -157,6 +163,10 @@ int _DkGetCPUInfo(PAL_CPU_INFO* ci) {
 
     int flen = 0, fmax = 80;
     char* flags = malloc(fmax);
+    if (!flags) {
+        rv = -PAL_ERROR_NOMEM;
+        goto out_brand;
+    }
 
     for (int i = 0; i < 32; i++) {
         if (!g_cpu_flags[i])
@@ -166,6 +176,10 @@ int _DkGetCPUInfo(PAL_CPU_INFO* ci) {
             int len = strlen(g_cpu_flags[i]);
             if (flen + len + 1 > fmax) {
                 char* new_flags = malloc(fmax * 2);
+                if (!new_flags) {
+                    rv = -PAL_ERROR_NOMEM;
+                    goto out_flags;
+                }
                 memcpy(new_flags, flags, flen);
                 free(flags);
                 fmax *= 2;
@@ -186,6 +200,13 @@ int _DkGetCPUInfo(PAL_CPU_INFO* ci) {
                 "Warning: bogomips could not be retrieved, passing 0.0 to the application\n");
     }
 
+    return rv;
+out_flags:
+    free(flags);
+out_brand:
+    free(brand);
+out_vendor_id:
+    free(vendor_id);
     return rv;
 }
 

Pal/src/host/Linux-SGX/db_main.c

@@ -284,14 +284,21 @@ noreturn void pal_linux_main(char* uptr_libpal_uri, size_t libpal_uri_len, char*
     g_pal_sec.uid = sec_info.uid;
     g_pal_sec.gid = sec_info.gid;
 
-    int num_cpus = sec_info.num_cpus;
-    if (num_cpus >= 1 && num_cpus <= (1 << 16)) {
-        g_pal_sec.num_cpus = num_cpus;
+    int online_logical_cores = sec_info.online_logical_cores;
+    if (online_logical_cores >= 1 && online_logical_cores <= (1 << 16)) {
+        g_pal_sec.online_logical_cores = online_logical_cores;
     } else {
-        SGX_DBG(DBG_E, "Invalid sec_info.num_cpus: %d\n", num_cpus);
+        SGX_DBG(DBG_E, "Invalid sec_info.online_logical_cores: %d\n", online_logical_cores);
         ocall_exit(1, /*is_exitgroup=*/true);
     }
 
+    if (sec_info.physical_cores_per_socket <= 0) {
+        SGX_DBG(DBG_E, "Invalid sec_info.physical_cores_per_socket: %ld\n",
+                sec_info.physical_cores_per_socket);
+        ocall_exit(1, /*is_exitgroup=*/true);
+    }
+    g_pal_sec.physical_cores_per_socket = sec_info.physical_cores_per_socket;
+
     /* set up page allocator and slab manager */
     init_slab_mgr(g_page_size);
     init_untrusted_slab_mgr();
@@ -337,6 +344,20 @@ noreturn void pal_linux_main(char* uptr_libpal_uri, size_t libpal_uri_len, char*
 
     SET_ENCLAVE_TLS(ready_for_exceptions, 1UL);
 
+    /* Allocate enclave memory to store "logical core -> socket" mappings */
+    int* cpu_socket = (int*)malloc(online_logical_cores * sizeof(int));
+    if (!cpu_socket) {
+        SGX_DBG(DBG_E, "Allocation for logical core -> socket mappings failed\n");
+        ocall_exit(1, /*is_exitgroup=*/true);
+    }
+
+    if (!sgx_copy_to_enclave(cpu_socket, online_logical_cores * sizeof(int), sec_info.cpu_socket,
+                             online_logical_cores * sizeof(int))) {
+        SGX_DBG(DBG_E, "Copying cpu_socket into the enclave failed\n");
+        ocall_exit(1, /*is_exitgroup=*/true);
+    }
+    g_pal_sec.cpu_socket = cpu_socket;
+
     /* initialize master key (used for pipes' encryption for all enclaves of an application); it
      * will be overwritten below in init_child_process() with inherited-from-parent master key if
      * this enclave is child */

Pal/src/host/Linux-SGX/pal_security.h

@@ -36,8 +36,9 @@ struct pal_sec {
     /* additional information */
     PAL_SEC_STR pipe_prefix;
 
-    /* Need to pass in the number of cores */
-    PAL_NUM num_cpus;
+    PAL_NUM online_logical_cores;
+    PAL_NUM physical_cores_per_socket;
+    int* cpu_socket;
 
 #ifdef DEBUG
     PAL_BOL in_gdb;

Pal/src/host/Linux-SGX/sgx_main.c

@@ -669,27 +669,30 @@ static int load_manifest(int fd, struct config_store** config_ptr) {
     return ret;
 }
 
-/*
- * Returns the number of online CPUs read from /sys/devices/system/cpu/online, -errno on failure.
- * Understands complex formats like "1,3-5,6".
+/* Opens a pseudo-file describing HW resources such as online CPUs and counts the number of
+ * HW resources present in the file (if count == true) or simply reads the integer stored in the
+ * file (if count == false). For example on a single-core machine, calling this function on
+ * `/sys/devices/system/cpu/online` with count == true will return 1 and 0 with count == false.
+ * Returns UNIX error code on failure.
+ * N.B: Understands complex formats like "1,3-5,6" when called with count == true.
  */
-static int get_cpu_count(void) {
-    int fd = INLINE_SYSCALL(open, 3, "/sys/devices/system/cpu/online", O_RDONLY | O_CLOEXEC, 0);
-    if (fd < 0)
-        return unix_to_pal_error(ERRNO(fd));
+static int get_hw_resource(const char* filename, bool count) {
+    int fd = INLINE_SYSCALL(open, 3, filename, O_RDONLY | O_CLOEXEC, 0);
+    if (IS_ERR(fd))
+        return -ERRNO(fd);
 
     char buf[64];
     int ret = INLINE_SYSCALL(read, 3, fd, buf, sizeof(buf) - 1);
-    if (ret < 0) {
-        INLINE_SYSCALL(close, 1, fd);
-        return unix_to_pal_error(ERRNO(ret));
-    }
+    INLINE_SYSCALL(close, 1, fd);
+    if (IS_ERR(ret))
+        return -ERRNO(ret);
 
     buf[ret] = '\0'; /* ensure null-terminated buf even in partial read */
 
     char* end;
     char* ptr = buf;
-    int cpu_count = 0;
+    int resource_cnt = 0;
+    int retval = -ENOENT;
     while (*ptr) {
         while (*ptr == ' ' || *ptr == '\t' || *ptr == ',')
             ptr++;
@@ -698,23 +701,31 @@ static int get_cpu_count(void) {
         if (ptr == end)
             break;
 
+        /* caller wants to read an int stored in the file */
+        if (!count) {
+            if (*end == '\n' || *end == '\0')
+                retval = firstint;
+            return retval;
+        }
+
+        /* caller wants to count the number of HW resources */
         if (*end == '\0' || *end == ',' || *end == '\n') {
-            /* single CPU index, count as one more CPU */
-            cpu_count++;
+            /* single HW resource index, count as one more */
+            resource_cnt++;
         } else if (*end == '-') {
-            /* CPU range, count how many CPUs in range */
+            /* HW resource range, count how many HW resources are in range */
             ptr = end + 1;
             int secondint = (int)strtol(ptr, &end, 10);
             if (secondint > firstint)
-                cpu_count += secondint - firstint + 1; // inclusive (e.g., 0-7, or 8-16)
+                resource_cnt += secondint - firstint + 1; // inclusive (e.g., 0-7, or 8-16)
         }
         ptr = end;
     }
 
-    INLINE_SYSCALL(close, 1, fd);
-    if (cpu_count == 0)
-        return -PAL_ERROR_STREAMNOTEXIST;
-    return cpu_count;
+    if (count && resource_cnt > 0)
+        retval = resource_cnt;
+
+    return retval;
 }
 
 /* Warning: This function does not free up resources on failure - it assumes that the whole process
@@ -743,11 +754,53 @@ static int load_enclave(struct pal_enclave* enclave, int manifest_fd, char* mani
     pal_sec->pid = INLINE_SYSCALL(getpid, 0);
     pal_sec->uid = INLINE_SYSCALL(getuid, 0);
     pal_sec->gid = INLINE_SYSCALL(getgid, 0);
-    int num_cpus = get_cpu_count();
-    if (num_cpus < 0) {
-        return num_cpus;
+
+    /* we cannot use CPUID(0xb) because it counts even disabled-by-BIOS cores (e.g. HT cores);
+     * instead extract info on total number of logical cores, number of physical cores,
+     * SMT support etc. by parsing sysfs pseudo-files */
+    int online_logical_cores = get_hw_resource("/sys/devices/system/cpu/online", /*count=*/true);
+    if (online_logical_cores < 0)
+        return online_logical_cores;
+    pal_sec->online_logical_cores = online_logical_cores;
+
+    int possible_logical_cores = get_hw_resource("/sys/devices/system/cpu/possible",
+                                                 /*count=*/true);
+    /* TODO: correctly support offline cores */
+    if (possible_logical_cores > 0 && possible_logical_cores > online_logical_cores) {
+         printf("Warning: some CPUs seem to be offline; Graphene doesn't take this into account "
+                "which may lead to subpar performance\n");
+    }
+
+
+    int core_siblings = get_hw_resource("/sys/devices/system/cpu/cpu0/topology/core_siblings_list",
+                                        /*count=*/true);
+    if (core_siblings < 0)
+        return core_siblings;
+
+    int smt_siblings = get_hw_resource("/sys/devices/system/cpu/cpu0/topology/thread_siblings_list",
+                                       /*count=*/true);
+    if (smt_siblings < 0)
+        return smt_siblings;
+    pal_sec->physical_cores_per_socket = core_siblings / smt_siblings;
+
+    /* array of "logical core -> socket" mappings */
+    int* cpu_socket = (int*)malloc(online_logical_cores * sizeof(int));
+    if (!cpu_socket)
+        return -ENOMEM;
+
+    char filename[128];
+    for (int idx = 0; idx < online_logical_cores; idx++) {
+        snprintf(filename, sizeof(filename),
+                 "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", idx);
+        cpu_socket[idx] = get_hw_resource(filename, /*count=*/false);
+        if (cpu_socket[idx] < 0) {
+            SGX_DBG(DBG_E, "Cannot read %s\n", filename);
+            ret = cpu_socket[idx];
+            free(cpu_socket);
+            return ret;
+        }
     }
-    pal_sec->num_cpus = num_cpus;
+    pal_sec->cpu_socket = cpu_socket;
 
 #ifdef DEBUG
     size_t env_i = 0;