[CI] Add new Jenkins pipeline for EDMM testing

Signed-off-by: Borys Popławski <borysp@invisiblethingslab.com>
gramineproject · Jan 9, 2023 · 1b1242f · 1b1242f
1 parent 75bf81a
commit 1b1242f
Show file tree

Hide file tree

Showing 60 changed files with 158 additions and 18 deletions.
diff --git a/.ci/lib/stage-build-sgx.jenkinsfile b/.ci/lib/stage-build-sgx.jenkinsfile
@@ -16,6 +16,13 @@ stage('build') {
     if (env.CC == 'clang') {
         env.MESON_OPTIONS += ' -Dmusl=disabled'
     }
+    if (env.RA_TYPE == 'dcap') {
+        env.MESON_OPTIONS += ' -Ddcap=enabled'
+    }
+
+    if (env.SGX_DRIVER == null) {
+        env.SGX_DRIVER = 'oot'
+    }
 
     try {
         sh '''
@@ -26,7 +33,7 @@ stage('build') {
                 -Ddirect=disabled \
                 -Dsgx=enabled \
                 -Dtests=enabled \
-                -Dsgx_driver=oot \
+                -Dsgx_driver="$SGX_DRIVER" \
                 $MESON_OPTIONS
             ninja -vC build/
         '''

diff --git a/.ci/lib/stage-test-sgx.jenkinsfile b/.ci/lib/stage-test-sgx.jenkinsfile
@@ -6,6 +6,11 @@ stage('test-sgx') {
             .ci/check-no-syscall.sh "$GRAMINE_PKGLIBDIR"/runtime/musl/libc.so
         fi
     '''
+
+    if (env.RA_TYPE == null) {
+        env.RA_TYPE = 'epid'
+    }
+
     timeout(time: 5, unit: 'MINUTES') {
         sh '''
             cd CI-Examples/helloworld
@@ -16,7 +21,7 @@ stage('test-sgx') {
     timeout(time: 5, unit: 'MINUTES') {
         sh '''
             cd CI-Examples/python
-            make ${MAKEOPTS} RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid}
+            make ${MAKEOPTS} RA_TYPE=$RA_TYPE RA_CLIENT_SPID=${ra_client_spid}
             make ${MAKEOPTS} check
         '''
     }
@@ -107,27 +112,42 @@ stage('test-sgx') {
     timeout(time: 5, unit: 'MINUTES') {
         sh '''
             cd CI-Examples/ra-tls-mbedtls
-            if [ "${ra_client_spid}" != "" ] && [ "${ra_client_key}" != "" ]; \
-            then \
-                make check_epid RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
-                    RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
-                make check_epid_fail RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
-                    RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
+            if [ "${RA_TYPE}" = "epid" ]; then \
+                if [ "${ra_client_spid}" != "" ] && [ "${ra_client_key}" != "" ]; \
+                then \
+                    make check_epid RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
+                        RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
+                    make check_epid_fail RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
+                        RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
+                else \
+                    echo "Failure: no ra_client_spid and/or ra_client_key!"; \
+                    exit 1; \
+                fi \
+            elif [ "${RA_TYPE}" = "dcap" ]; then \
+                make check_dcap RA_TYPE=dcap; \
+                make check_dcap_fail RA_TYPE=dcap; \
             else \
-                echo "Failure: no ra_client_spid and/or ra_client_key!"; \
+                echo "Invalid RA_TYPE env variable: ${RA_TYPE}"; \
                 exit 1; \
             fi
         '''
     }
     timeout(time: 5, unit: 'MINUTES') {
         sh '''
             cd CI-Examples/ra-tls-secret-prov
-            if [ "${ra_client_spid}" != "" ] && [ "${ra_client_key}" != "" ]; \
-            then \
-                make check_epid RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
-                    RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
+            if [ "${RA_TYPE}" = "epid" ]; then \
+                if [ "${ra_client_spid}" != "" ] && [ "${ra_client_key}" != "" ]; \
+                then \
+                    make check_epid RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
+                        RA_TLS_EPID_API_KEY=${ra_client_key} RA_CLIENT_LINKABLE=0; \
+                else \
+                    echo "Failure: no ra_client_spid and/or ra_client_key!"; \
+                    exit 1; \
+                fi \
+            elif [ "${RA_TYPE}" = "dcap" ]; then \
+                make check_dcap RA_TYPE=dcap; \
             else \
-                echo "Failure: no ra_client_spid and/or ra_client_key!"; \
+                echo "Invalid RA_TYPE env variable: ${RA_TYPE}"; \
                 exit 1; \
             fi
         '''

diff --git a/.ci/lib/stage-test.jenkinsfile b/.ci/lib/stage-test.jenkinsfile
@@ -1,4 +1,8 @@
 stage('test') {
+    if (env.RA_TYPE == null) {
+        env.RA_TYPE = 'epid'
+    }
+
     timeout(time: 15, unit: 'MINUTES') {
         try {
             sh '''
@@ -15,7 +19,7 @@ stage('test') {
         try {
             sh '''
                 cd libos/test/regression
-                RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} gramine-test build -v
+                RA_TYPE=$RA_TYPE RA_CLIENT_SPID=${ra_client_spid} gramine-test build -v
                 python3 -m pytest -v --junit-xml libos-regression.xml
             '''
         } finally {
@@ -31,7 +35,7 @@ stage('test') {
                     cd libos/test/regression
                     # For some unknown reason it fails without this clean on sgx-18.04 pipeline
                     gramine-test clean
-                    RA_TYPE=epid RA_CLIENT_SPID=${ra_client_spid} \
+                    RA_TYPE=$RA_TYPE RA_CLIENT_SPID=${ra_client_spid} \
                         gramine-test -n tests_musl.toml build -v
                     python3 -m pytest -v --junit-xml libos-regression-musl.xml
                 '''

diff --git a/.ci/linux-sgx-edmm.jenkinsfile b/.ci/linux-sgx-edmm.jenkinsfile
@@ -0,0 +1,30 @@
+node('sgx-edmm') {
+    checkout scm
+
+    env.SGX = '1'
+    env.SGX_DRIVER = 'upstream'
+    env.EDMM = '1'
+    env.RA_TYPE = 'dcap'
+
+    load '.ci/lib/config-docker.jenkinsfile'
+
+    env.DOCKER_ARGS_SGX += '''
+        --volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro
+        --add-host host.docker.internal:host-gateway
+    '''
+
+    docker.build(
+        "local:${env.BUILD_TAG}",
+        '-f .ci/ubuntu20.04.dockerfile .'
+    ).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") {
+        load '.ci/lib/config.jenkinsfile'
+        load '.ci/lib/config-release.jenkinsfile'
+
+        load '.ci/lib/stage-lint.jenkinsfile'
+        load '.ci/lib/stage-clean-check-prepare.jenkinsfile'
+        load '.ci/lib/stage-build-sgx.jenkinsfile'
+        load '.ci/lib/stage-test.jenkinsfile'
+        load '.ci/lib/stage-test-sgx.jenkinsfile'
+        load '.ci/lib/stage-clean-check.jenkinsfile'
+    }
+}
diff --git a/.ci/ubuntu20.04.dockerfile b/.ci/ubuntu20.04.dockerfile
@@ -72,6 +72,18 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     zlib1g \
     zlib1g-dev
 
+# Needed by DCAP attestation e.g. in "CI-Examples/ra-tls-mbedtls"
+RUN curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -
+RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' > /etc/apt/sources.list.d/intel-sgx.list
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libsgx-dcap-default-qpl \
+    libsgx-dcap-quote-verify-dev \
+    libsgx-urts
+
+# set up PCCS connection configuration
+RUN sed -i -e 's/localhost/host.docker.internal/g' /etc/sgx_default_qcnl.conf \
+    && sed -i -e 's/"use_secure_cert": true/"use_secure_cert": false/' /etc/sgx_default_qcnl.conf
+
 # Install wrk2 benchmark. This benchmark is used in `benchmark-http.sh`.
 RUN git clone https://github.com/giltene/wrk2.git \
     && cd wrk2 \

diff --git a/CI-Examples/bash/manifest.template b/CI-Examples/bash/manifest.template
@@ -19,6 +19,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sgx.enclave_size = "512M"
 sgx.max_threads = 4

diff --git a/CI-Examples/blender/blender.manifest.template b/CI-Examples/blender/blender.manifest.template
@@ -21,6 +21,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sys.stack.size = "8M"
 sgx.enclave_size = "2048M"

diff --git a/CI-Examples/busybox/busybox.manifest.template b/CI-Examples/busybox/busybox.manifest.template
@@ -21,6 +21,7 @@ fs.mounts = [
 sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/helloworld/helloworld.manifest.template b/CI-Examples/helloworld/helloworld.manifest.template
@@ -12,6 +12,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 
 sgx.trusted_files = [

diff --git a/CI-Examples/lighttpd/lighttpd.manifest.template b/CI-Examples/lighttpd/lighttpd.manifest.template
@@ -20,6 +20,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sgx.enclave_size = "256M"
 sgx.max_threads = 3

diff --git a/CI-Examples/memcached/memcached.manifest.template b/CI-Examples/memcached/memcached.manifest.template
@@ -24,6 +24,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sgx.max_threads = 16
 

diff --git a/CI-Examples/nginx/nginx.manifest.template b/CI-Examples/nginx/nginx.manifest.template
@@ -26,6 +26,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sgx.enclave_size = "512M"
 sgx.max_threads = 4

diff --git a/CI-Examples/python/python.manifest.template b/CI-Examples/python/python.manifest.template
@@ -33,6 +33,7 @@ sys.stack.size = "2M"
 sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 sgx.enclave_size = "1G"
 sgx.max_threads = 32

diff --git a/CI-Examples/ra-tls-mbedtls/client.manifest.template b/CI-Examples/ra-tls-mbedtls/client.manifest.template
@@ -25,6 +25,7 @@ fs.mounts = [
 sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.enclave_size = "512M"
 sgx.max_threads = 4
 

diff --git a/CI-Examples/ra-tls-mbedtls/server.manifest.template b/CI-Examples/ra-tls-mbedtls/server.manifest.template
@@ -20,6 +20,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/ra-tls-secret-prov/secret_prov/client.manifest.template b/CI-Examples/ra-tls-secret-prov/secret_prov/client.manifest.template
@@ -20,6 +20,7 @@ sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.enclave_size = "512M"
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/ra-tls-secret-prov/secret_prov_minimal/client.manifest.template b/CI-Examples/ra-tls-secret-prov/secret_prov_minimal/client.manifest.template
@@ -24,6 +24,7 @@ sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.enclave_size = "512M"
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/ra-tls-secret-prov/secret_prov_pf/client.manifest.template b/CI-Examples/ra-tls-secret-prov/secret_prov_pf/client.manifest.template
@@ -26,6 +26,7 @@ sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.enclave_size = "512M"
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/redis/redis-server.manifest.template b/CI-Examples/redis/redis-server.manifest.template
@@ -70,6 +70,13 @@ sgx.debug = true
 # typical Redis workloads.
 sgx.enclave_size = "1024M"
 
+# Enable Enclave Dynamic Memory Management (EDMM) feature based on EDMM
+# environment variable. This allows for addition of pages to enclave in runtime,
+# instead of allocating them upfront at startup. If this feature is enabled,
+# `sgx.enclave_size` above describes a maximal enclave size and can usually be
+# increased without negative consequences (it does not impact startup time).
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
+
 # Set maximum number of in-enclave threads (somewhat arbitrarily) to 8. Recall
 # that SGX v1 requires to specify the maximum number of simultaneous threads at
 # enclave creation time.

diff --git a/CI-Examples/rust/rust-hyper-http-server.manifest.template b/CI-Examples/rust/rust-hyper-http-server.manifest.template
@@ -18,6 +18,7 @@ fs.mounts = [
 ]
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.nonpie_binary = true
 
 sgx.trusted_files = [

diff --git a/CI-Examples/sqlite/manifest.template b/CI-Examples/sqlite/manifest.template
@@ -27,6 +27,7 @@ fs.mounts = [
 fs.insecure__keys.default = "ffeeddccbbaa99887766554433221100"
 
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.enclave_size = "256M"
 sgx.max_threads = 4
 

diff --git a/libos/test/abi/x86_64/manifest.template b/libos/test/abi/x86_64/manifest.template
@@ -8,6 +8,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.max_threads = 4
 
 sgx.trusted_files = [

diff --git a/libos/test/abi/x86_64/stack_arg.manifest.template b/libos/test/abi/x86_64/stack_arg.manifest.template
@@ -13,6 +13,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.max_threads = 4
 
 sgx.trusted_files = [

diff --git a/libos/test/abi/x86_64/stack_env.manifest.template b/libos/test/abi/x86_64/stack_env.manifest.template
@@ -13,6 +13,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.max_threads = 4
 
 sgx.trusted_files = [

diff --git a/libos/test/fs/manifest.template b/libos/test/fs/manifest.template
@@ -22,6 +22,7 @@ fs.insecure__keys.default = "ffeeddccbbaa99887766554433221100"
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 sgx.max_threads = 16
 
 sgx.allowed_files = [

diff --git a/libos/test/ltp/manifest.template b/libos/test/ltp/manifest.template
@@ -21,6 +21,7 @@ sys.brk.max_size = "32M"
 sys.stack.size = "4M"
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.allowed_files = [
   "file:/tmp",

diff --git a/libos/test/regression/argv_from_file.manifest.template b/libos/test/regression/argv_from_file.manifest.template
@@ -13,6 +13,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.allowed_files = [
   "file:argv_test_input",

diff --git a/libos/test/regression/argv_from_manifest.manifest.template b/libos/test/regression/argv_from_manifest.manifest.template
@@ -20,6 +20,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/libos/test/regression/attestation.manifest.template b/libos/test/regression/attestation.manifest.template
@@ -13,6 +13,7 @@ fs.insecure__keys.default = "ffeeddccbbaa99887766554433221100"
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = "{{ env.get('RA_TYPE', 'none') }}"
 sgx.ra_client_spid = "{{ env.get('RA_CLIENT_SPID', '') }}"

diff --git a/libos/test/regression/attestation_deprecated_syntax.manifest.template b/libos/test/regression/attestation_deprecated_syntax.manifest.template
@@ -15,6 +15,7 @@ sgx.insecure__protected_files_key = "ffeeddccbbaa99887766554433221100"
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.remote_attestation = true
 sgx.ra_client_spid = "{{ env.get('RA_CLIENT_SPID', '') }}"

diff --git a/libos/test/regression/bootstrap_cpp.manifest.template b/libos/test/regression/bootstrap_cpp.manifest.template
@@ -16,6 +16,7 @@ fs.mounts = [
 sgx.max_threads = 8
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/libos/test/regression/debug_log_file.manifest.template b/libos/test/regression/debug_log_file.manifest.template
@@ -15,6 +15,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
+sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",