From c90a38e76e2fdfae36a8b4b4732782a9c265dce6 Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Thu, 6 Jan 2022 06:37:32 -0800 Subject: [PATCH] [Pal/Linux-SGX] Add `sgx.disable_[cpu-feature]` manifest options This commit adds three new manifest options: `sgx.disable_avx`, `sgx.disable_avx512`, `sgx.disable_amx`. Setting each of these options to `true` disables the corresponding CPU feature inside the SGX enclave even if this CPU feature is available on the system: this may improve enclave performance because this CPU feature will *not* be saved and restored during enclave entry/exit. Signed-off-by: Dmitrii Kuvaiskii --- Documentation/devel/performance.rst | 11 ++++++++ Documentation/manifest-syntax.rst | 38 ++++++++++++++++++++++---- Pal/src/host/Linux-SGX/sgx_framework.c | 12 +++++++- Pal/src/host/Linux-SGX/sgx_internal.h | 5 ++++ Pal/src/host/Linux-SGX/sgx_main.c | 25 +++++++++++++++++ 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/Documentation/devel/performance.rst b/Documentation/devel/performance.rst index 2d084d11a9..f329ab687b 100644 --- a/Documentation/devel/performance.rst +++ b/Documentation/devel/performance.rst @@ -233,6 +233,17 @@ platform and use very slow functions, leading to 10-100x overhead over native your case, enable the features in the manifest, e.g., set ``sgx.require_avx = true``. +Gramine also allows to explicitly disable not-security-critical CPU features +using the following manifest options: ``sgx.disable_avx``, +``sgx.disable_avx512``, ``sgx.disable_amx``. By default, all of these options +are set to ``false`` -- this means that Gramine will enable the CPU feature if +available on the system. Setting each of these options to ``true`` disables the +corresponding CPU feature inside the SGX enclave even if this CPU feature is +available on the system: this may improve enclave performance because this CPU +feature will *not* be saved and restored during enclave entry/exit. But be aware +that if the graminized application relies on this CPU feature, the application +will crash with "illegal instruction". + For more information on SGX logic regarding optional CPU features, see the Intel Software Developer Manual, Table 38-3 ("Layout of ATTRIBUTES Structure") under the SGX section. diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst index 5d89b5bfe0..3f6a4b4b85 100644 --- a/Documentation/manifest-syntax.rst +++ b/Documentation/manifest-syntax.rst @@ -539,11 +539,39 @@ Optional CPU features (AVX, AVX512, MPX, PKRU, AMX) sgx.require_amx = [true|false] (Default: false) -This syntax ensures that the CPU features are available and enabled for the -enclave. If the options are set in the manifest but the features are unavailable -on the platform, enclave initialization will fail. If the options are unset, -enclave initialization will succeed even if these features are unavailable on -the platform. + sgx.disable_avx = [true|false] + sgx.disable_avx512 = [true|false] + sgx.disable_amx = [true|false] + (Default: false) + +The ``sgx.require_[feature]`` syntax ensures that the corresponding CPU feature +is available and enabled for the SGX enclave. If the option is set in the +manifest but the corresponding CPU feature is unavailable on the platform, +enclave initialization will fail. If the option is unset, enclave initialization +will succeed even if the corresponding feature is unavailable on the platform. + +The ``sgx.disable_[feature]`` syntax disables the corresponding CPU feature +inside the SGX enclave even if this CPU feature is available on the platform: +this may improve enclave performance because this CPU feature will *not* be +saved and restored during enclave entry/exit. This syntax is provided to improve +performance of applications that are known to *not* rely on certain CPU +features. Be aware that if the application relies on some disabled CPU features, +the application will fail with SIGILL ("illegal instruction"). For example, if +the application is built with AVX support, and AVX is disabled in the manifest, +the application will crash. Only not-security-critical CPU features may be +disabled (currently these are AVX, AVX512 and AMX). + +It is meaningless to set a CPU feature as both required and disabled. Currently +Gramine doesn't disallow this, but the feature will be disabled in such case. +For example, setting both ``sgx.require_avx = true`` and ``sgx.disable_avx = +true`` will result in the SGX enclave running with AVX disabled. + +In case of doubt, it is recommended to keep the default values for these +features (e.g. ``sgx.require_avx = false`` and ``sgx.disable_avx = +false``). In this case, Gramine auto-detects the corresponding CPU features on +the platform and enables them if available, regardless of whether the +application uses them or not. + ISV Product ID and SVN ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Pal/src/host/Linux-SGX/sgx_framework.c b/Pal/src/host/Linux-SGX/sgx_framework.c index 482561c2c0..7f0adf5ee6 100644 --- a/Pal/src/host/Linux-SGX/sgx_framework.c +++ b/Pal/src/host/Linux-SGX/sgx_framework.c @@ -58,7 +58,9 @@ int read_enclave_token(int token_file, sgx_arch_token_t* token) { return bytes; #ifdef SGX_DCAP - log_debug("Read dummy DCAP token"); + log_debug("Read dummy DCAP token (only `attr` field is used):"); + log_debug(" attr.flags: 0x%016lx", token->body.attributes.flags); + log_debug(" attr.xfrm: 0x%016lx", token->body.attributes.xfrm); #else log_debug("Read token:"); log_debug(" valid: 0x%08x", token->body.valid); @@ -118,6 +120,14 @@ int create_enclave(sgx_arch_secs_t* secs, sgx_arch_token_t* token) { secs->misc_select = token->masked_misc_select_le; memcpy(&secs->attributes, &token->body.attributes, sizeof(sgx_attributes_t)); + /* disable not-security-critical HW features for XSAVE/AEX performance; see also sgx_arch.h */ + if (g_pal_enclave.avx_disabled) + secs->attributes.xfrm &= ~SGX_XFRM_AVX; + if (g_pal_enclave.avx512_disabled) + secs->attributes.xfrm &= ~SGX_XFRM_AVX512; + if (g_pal_enclave.amx_disabled) + secs->attributes.xfrm &= ~SGX_XFRM_AMX; + /* Do not initialize secs->mr_signer and secs->mr_enclave here as they are * not used by ECREATE to populate the internal SECS. SECS's mr_enclave is * computed dynamically and SECS's mr_signer is populated based on the diff --git a/Pal/src/host/Linux-SGX/sgx_internal.h b/Pal/src/host/Linux-SGX/sgx_internal.h index 032ad048bc..6e581891b3 100644 --- a/Pal/src/host/Linux-SGX/sgx_internal.h +++ b/Pal/src/host/Linux-SGX/sgx_internal.h @@ -53,6 +53,11 @@ struct pal_enclave { bool use_epid_attestation; /* Valid only if `remote_attestation_enabled` is true, selects * EPID/DCAP attestation scheme. */ + /* disable not-security-critical HW features (for performance of XSAVE/XRSTOR/AEX) */ + bool avx_disabled; + bool avx512_disabled; + bool amx_disabled; + /* files */ int sigfile; int token; diff --git a/Pal/src/host/Linux-SGX/sgx_main.c b/Pal/src/host/Linux-SGX/sgx_main.c index fe55d9dd9f..5ea44673ae 100644 --- a/Pal/src/host/Linux-SGX/sgx_main.c +++ b/Pal/src/host/Linux-SGX/sgx_main.c @@ -727,6 +727,31 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info) /* EPID is used if SPID is a non-empty string in manifest, otherwise DCAP/ECDSA */ enclave_info->use_epid_attestation = sgx_ra_client_spid_str && strlen(sgx_ra_client_spid_str); + /* check if not-security-critical HW features should be disabled for XSAVE/AEX performance */ + ret = toml_bool_in(manifest_root, "sgx.disable_avx", /*defaultval=*/false, + &enclave_info->avx_disabled); + if (ret < 0) { + log_error("Cannot parse 'sgx.disable_avx' (the value must be `true` or `false`)"); + ret = -EINVAL; + goto out; + } + + ret = toml_bool_in(manifest_root, "sgx.disable_avx512", /*defaultval=*/false, + &enclave_info->avx512_disabled); + if (ret < 0) { + log_error("Cannot parse 'sgx.disable_avx512' (the value must be `true` or `false`)"); + ret = -EINVAL; + goto out; + } + + ret = toml_bool_in(manifest_root, "sgx.disable_amx", /*defaultval=*/false, + &enclave_info->amx_disabled); + if (ret < 0) { + log_error("Cannot parse 'sgx.disable_amx' (the value must be `true` or `false`)"); + ret = -EINVAL; + goto out; + } + ret = toml_string_in(manifest_root, "sgx.profile.enable", &profile_str); if (ret < 0) { log_error("Cannot parse 'sgx.profile.enable' "