Workload parser that verifies the results and generates XML results

jinengandhi-intel · Aug 25, 2021 · fe4d075 · fe4d075
1 parent 04addfe
commit fe4d075
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 2 deletions.
diff --git a/ci/linux-direct-centos8.2-gcc-release.jenkinsfile b/ci/linux-direct-centos8.2-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node(node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
 
             sh 'sed -i -e \'s/dent != g_dentry_root && dent->state & DENTRY_VALID/0/\' \

diff --git a/ci/linux-direct-ubuntu18.04-gcc-release.jenkinsfile b/ci/linux-direct-ubuntu18.04-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node(node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
 
             sh 'sed -i -e \'s/dent != g_dentry_root && dent->state & DENTRY_VALID/0/\' \

diff --git a/ci/linux-direct-ubuntu20.04-gcc-release.jenkinsfile b/ci/linux-direct-ubuntu20.04-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node(node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
 
             sh 'sed -i -e \'s/dent != g_dentry_root && dent->state & DENTRY_VALID/0/\' \

diff --git a/ci/linux-sgx-centos8.2-gcc-release.jenkinsfile b/ci/linux-sgx-centos8.2-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node (node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
             env.SGX = '1'
 

diff --git a/ci/linux-sgx-ubuntu18.04-gcc-release.jenkinsfile b/ci/linux-sgx-ubuntu18.04-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node (node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
             env.SGX = '1'
 

diff --git a/ci/linux-sgx-ubuntu20.04-gcc-release.jenkinsfile b/ci/linux-sgx-ubuntu20.04-gcc-release.jenkinsfile
@@ -16,6 +16,7 @@ node(node_label) {
             sh 'cp -rf $WORKSPACE/ltp_src LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/ltp_config/* LibOS/shim/test/ltp/'
             sh 'cp -rf $WORKSPACE/stress-ng Examples/'
+            sh 'cp -f $WORKSPACE/test_workloads.py . '
             env.WORKSPACE = env.WORKSPACE + "/graphene"
             env.SGX = '1'
 

diff --git a/ci/stage-test-direct.jenkinsfile b/ci/stage-test-direct.jenkinsfile
@@ -46,6 +46,7 @@ stage('test-direct') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/bash
+                sed -i '/@rm OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make ${MAKEOPTS} regression
             '''
@@ -59,6 +60,7 @@ stage('test-direct') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/curl
+                sed -i '/@rm OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make ${MAKEOPTS} check
             '''
@@ -111,7 +113,7 @@ stage('test-direct') {
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make start-graphene-server &
                 ../../Scripts/wait_for_server 5 127.0.0.1 6379
-                ./src/src/redis-benchmark
+                ./src/src/redis-benchmark 2>&1 | tee OUTPUT
             '''
         }
     } catch (Exception e){
@@ -180,6 +182,7 @@ stage('test-direct') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/r
+                sed -i '/@$(RM) OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make check
             '''
@@ -188,4 +191,15 @@ stage('test-direct') {
         build_ok = false
         sh 'echo "R Example Test Failed"'
     }
+
+    try {
+        timeout(time: 2, unit: 'MINUTES') {
+            sh 'python3 -m pytest -v --junit-xml workload-regression.xml test_workloads.py'
+        }
+    } catch (Exception e){
+        build_ok = true
+        sh 'echo "Workload parser failed"'
+    } finally {
+        junit 'workload-regression.xml'
+    }
 }
diff --git a/ci/stage-test-sgx.jenkinsfile b/ci/stage-test-sgx.jenkinsfile
@@ -47,6 +47,7 @@ stage('test-sgx') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/bash
+                sed -i '/@rm OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make ${MAKEOPTS} SGX=1 regression
             '''
@@ -60,6 +61,7 @@ stage('test-sgx') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/curl
+                sed -i '/@rm OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make ${MAKEOPTS} SGX=1 check
             '''
@@ -115,7 +117,7 @@ stage('test-sgx') {
                 make ${MAKEOPTS} ${ARCH_LIB_OPT}
                 make ${MAKEOPTS} SGX=1 start-graphene-server &
                 ../../Scripts/wait_for_server 60 127.0.0.1 6379
-                ./src/src/redis-benchmark
+                ./src/src/redis-benchmark 2>&1 | tee OUTPUT
             '''
         }
     } catch (Exception e){
@@ -189,6 +191,7 @@ stage('test-sgx') {
         timeout(time: 5, unit: 'MINUTES') {
             sh '''
                 cd Examples/r
+                sed -i '/@$(RM) OUTPUT/d' Makefile
                 make ${MAKEOPTS} ${ARCH_LIB_OPT} all
                 make ${MAKEOPTS} SGX=1 check
             '''
@@ -234,4 +237,15 @@ stage('test-sgx') {
             fi
         '''
     }*/
+
+    try {
+        timeout(time: 2, unit: 'MINUTES') {
+            sh 'python3 -m pytest -v --junit-xml workload-regression.xml test_workloads.py'
+        }
+    } catch (Exception e){
+        build_ok = true
+        sh 'echo "Workload parser failed"'
+    } finally {
+        junit 'workload-regression.xml'
+    }
 }
diff --git a/test_workloads.py b/test_workloads.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import sys
+import logging
+import glob
+from os import path
+import os
+
+
+class Test_Workload_Results():
+    def test_bash_workload(self):
+        bash_result_file = open("Examples/bash/OUTPUT", "r")
+        bash_contents = bash_result_file.read()
+        assert("readlink" in bash_contents)
+
+    def test_curl_load(self):
+        curl_result_file = open("Examples/curl/OUTPUT", "r")
+        curl_contents = curl_result_file.read()
+        assert("Hello World" in curl_contents)
+
+    def test_python_workload(self):
+        python_result_file = open("Examples/python/TEST_STDOUT", "r")
+        python_contents = python_result_file.read()
+        assert("Success 1/4" in python_contents)
+        assert("Success 2/4" in python_contents)
+        assert("Success 3/4" in python_contents)
+        assert("Success 4/4" in python_contents)
+
+    def test_lightppd_workload(self):
+        for filename in glob.glob("Examples/lighttpd/result-*"):
+            lightppd_result_file = open(filename,"r")
+        lightppd_contents = lightppd_result_file.read()
+        assert("0,0" in lightppd_contents)
+
+    def test_nginx_workload(self):
+        for filename in glob.glob("Examples/nginx/result-*"):
+            nginx_result_file = open(filename,"r")
+        nginx_contents = nginx_result_file.read()
+        assert("0,0" in nginx_contents)
+
+    def test_blender(self):
+        blender_result_file = "Examples/blender/data/images/simple_scene.blend0001.png"
+        assert(path.exists(blender_result_file))
+
+    def test_r_load(self):
+        r_result_file = open("Examples/r/OUTPUT", "r")
+        r_contents = r_result_file.read()
+        assert("success" in r_contents)
+
+    def test_redis(self):
+        redis_result_file = open("Examples/redis/OUTPUT", "r")
+        redis_contents = redis_result_file.read()
+        assert(("PING_INLINE" in redis_contents) and ("MSET" in redis_contents))
+