Merge pull request #1 from distributed-system-analysis/master

catch up to distributed-system-analysis/pbench

README.md

@@ -1,4 +1,4 @@
-# pbench
+# Pbench [![Stories in Ready](https://badge.waffle.io/distributed-system-analysis/pbench.png?label=ready&title=Ready)](https://waffle.io/distributed-system-analysis/pbench)
 A benchmarking and performance analysis framework
 
 The code base includes three sub-systems. The first being the collection
@@ -31,6 +31,6 @@ Yes, we use Google Groups:
     https://groups.google.com/forum/#!forum/pbench
 
 ## Is there a place to track current and future work?
-Yes, we are using Trello for that:
+Yes, we are using Waffle.io for that:
 
-    https://trello.com/b/y3YJgHim/pbench
+    https://waffle.io/distributed-system-analysis/pbench

agent/base

@@ -95,15 +95,21 @@ fi
 
 function check_install_rpm {
     local this_rpm=$1
-    local this_version=$2
+    if [ ! -z "$2" ]; then
+    	local this_version="-$2"
+    else
+	local this_version=""
+    fi
+
     rpm_status=`rpm --query $this_rpm`
     if echo $rpm_status | grep -q "is not installed"; then
-	yum --debuglevel=0 install -y $this_rpm-$this_version > $pbench_log 2>&1
+	echo attempting to install ${this_rpm}$this_version
+	yum --debuglevel=0 install -y ${this_rpm}$this_version >> $pbench_log 2>&1
 	rc=$?
     else
 	local installed_rpm_version=`echo $rpm_status | awk -F- '{print $2"-"$3}'`
 	if [ "$this_version" != "$installed_rpm_version" ]; then
-	    yum --debuglevel=0 install -y $this_rpm-$this_version > $pbench_log 2>&1
+	    yum --debuglevel=0 install -y ${this_rpm}$this_version >> $pbench_log 2>&1
 	    rc=$?
 	else
 	    debug_log "$this_rpm has already been installed"

agent/config/pbench-agent.conf → agent/config/pbench-agent.conf.example

@@ -1,10 +1,10 @@
 [DEFAULT]
 pbench_install_dir = /opt/pbench-agent
-configtools_install_dir = /opt/configtools
+pbench_results_redirector = pbench.results.example.com
+pbench_web_server = pbench.example.com
 
 [pbench-agent]
 version = 0.33
-rpm-repo = %(pbench_repo_host)s:/pbench/repo/%%s
 pbench_run = /var/lib/pbench-agent
 pbench_log = %(pbench_run)s/pbench.log
 

agent/tool-scripts/README

@@ -1,10 +1,10 @@
-Scripts in this directory are generally called by start-tools, stop-tools, and postprocess-tools
-(found in ./util-scripts).  They are usually not called directly.
+Scripts in this directory are generally called by start-tools,stop-tools, and
+postprocess-tools (found in ./util-scripts).  They are usually not called
+directly.
 
 Note: many of these files are just link to the same file.
 
 #ls -li | sort -n
-90320298 -rw-r--r--. 1 root root 1771 Jul 23 16:05 README
 90320299 -rwxr-xr-x. 4 root root 6776 Jul 23 11:45 iostat
 90320299 -rwxr-xr-x. 4 root root 6776 Jul 23 11:45 mpstat
 90320299 -rwxr-xr-x. 4 root root 6776 Jul 23 11:45 pidstat
@@ -31,25 +31,27 @@ inode            ref-count
 
 These are all the same file:
 
-cpuacct
-kvm-spinlock
-kvmstat
-numastat
-proc-interrupts
-proc-sched_debug
-proc-vmstat
-sysfs
-virsh-migrate
+    cpuacct
+    kvm-spinlock
+    kvmstat
+    numastat
+    proc-interrupts
+    proc-sched_debug
+    proc-vmstat
+    sysfs
+    virsh-migrate
 
-And thse are the same file:
+And these are the same file:
 
-iostat
-mpstat
-pidstat
-sar
+    iostat
+    mpstat
+    pidstat
+    sar
 
 So, be mindful of editing one of these files, as it affects the other tools.
 
+The datalog directory: If a tool needs another script to log data, put it in
+this directory.
 
-The datalog directory:  If a tool needs another script to log data, put it in this directory
-The postprocess directory:  only postprocessing scripts go here
+The postprocess directory: Only postprocessing scripts, and their unit tests,
+go here.

agent/tool-scripts/postprocess/gold/uperf-0/stdout

@@ -1,5 +1,5 @@
 resp-file: response-times-172.168.10.88.txt
-average latency: 54.8069
-95th percentile latency: 56.8610
-99.995th percentile latency: 87.5680
-maximum latency: 1441.3400
+average latency: 55.2066
+95th percentile latency: 59.3970
+99.995th percentile latency: 96.1700
+maximum latency: 96.1700

agent/tool-scripts/postprocess/gold/uperf-0/summary-response-times.txt

@@ -1,6 +1,6 @@
-Number of writes: 1056503  Average response time for writes is: 6.7230us
-Number of reads: 1056502  Average response time for reads is: 48.0839us
-Average transaction latency: 54.8069us
-95th percentile transaction latency: 56.8610us
-99.995th percentile transaction latency: 87.5680us
-Maximum transaction latency: 1441.3400us
+Number of writes: 500  Average response time for writes is: 6.7818us
+Number of reads: 499  Average response time for reads is: 48.4248us
+Average transaction latency: 55.2066us
+95th percentile transaction latency: 59.3970us
+99.995th percentile transaction latency: 96.1700us
+Maximum transaction latency: 96.1700us

agent/tool-scripts/postprocess/samples/fio-0/result/reference-result/clients/localhost/fio_clat.1.log

@@ -0,0 +1,31 @@
+1000, 2888, 0, 0
+2000, 2885, 0, 0
+3000, 2887, 0, 0
+4000, 2891, 0, 0
+5000, 2889, 0, 0
+6000, 2887, 0, 0
+7000, 2889, 0, 0
+8000, 2888, 0, 0
+9000, 2896, 0, 0
+10000, 2896, 0, 0
+11000, 2893, 0, 0
+12000, 2896, 0, 0
+13000, 2898, 0, 0
+14000, 2887, 0, 0
+15000, 2887, 0, 0
+16000, 2890, 0, 0
+17000, 2886, 0, 0
+18000, 2887, 0, 0
+19000, 2886, 0, 0
+20000, 2890, 0, 0
+21000, 2885, 0, 0
+22000, 2886, 0, 0
+23000, 2887, 0, 0
+24000, 2887, 0, 0
+25000, 2888, 0, 0
+26000, 2889, 0, 0
+27000, 2882, 0, 0
+28000, 2888, 0, 0
+29000, 2892, 0, 0
+30000, 2897, 0, 0
+30002, 4159, 0, 0

agent/tool-scripts/postprocess/samples/fio-0/result/reference-result/clients/localhost/fio_lat.1.log

@@ -0,0 +1,31 @@
+1000, 2909, 0, 0
+2000, 2906, 0, 0
+3000, 2908, 0, 0
+4000, 2911, 0, 0
+5000, 2909, 0, 0
+6000, 2908, 0, 0
+7000, 2910, 0, 0
+8000, 2909, 0, 0
+9000, 2917, 0, 0
+10000, 2918, 0, 0
+11000, 2914, 0, 0
+12000, 2917, 0, 0
+13000, 2919, 0, 0
+14000, 2908, 0, 0
+15000, 2908, 0, 0
+16000, 2910, 0, 0
+17000, 2906, 0, 0
+18000, 2909, 0, 0
+19000, 2908, 0, 0
+20000, 2910, 0, 0
+21000, 2906, 0, 0
+22000, 2906, 0, 0
+23000, 2908, 0, 0
+24000, 2908, 0, 0
+25000, 2910, 0, 0
+26000, 2910, 0, 0
+27000, 2903, 0, 0
+28000, 2908, 0, 0
+29000, 2912, 0, 0
+30000, 2918, 0, 0
+30002, 4182, 0, 0

agent/tool-scripts/postprocess/samples/fio-0/result/reference-result/clients/localhost/fio_slat.1.log

@@ -0,0 +1,35 @@
+1000, 25, 0, 0
+2000, 23, 0, 0
+3003, 21, 0, 0
+4003, 20, 0, 0
+0, 20, 0, 0
+1000, 21, 0, 0
+2000, 21, 0, 0
+3000, 20, 0, 0
+4000, 20, 0, 0
+5000, 20, 0, 0
+6000, 21, 0, 0
+7000, 21, 0, 0
+8000, 20, 0, 0
+9000, 21, 0, 0
+10000, 21, 0, 0
+11000, 21, 0, 0
+12000, 21, 0, 0
+13000, 20, 0, 0
+14000, 20, 0, 0
+15000, 20, 0, 0
+16000, 20, 0, 0
+17000, 20, 0, 0
+18000, 21, 0, 0
+19000, 21, 0, 0
+20000, 19, 0, 0
+21000, 21, 0, 0
+22000, 20, 0, 0
+23000, 21, 0, 0
+24000, 20, 0, 0
+25000, 21, 0, 0
+26000, 21, 0, 0
+27000, 20, 0, 0
+28000, 20, 0, 0
+29000, 19, 0, 0
+30000, 21, 0, 0

agent/util-scripts/move-results

@@ -58,16 +58,16 @@ if [ -z "$results_webserver" ]; then
     exit 1
 fi
 
-ver=$(yum info installed pbench 2> /dev/null | grep Version | awk '{ print $3 }')
+ver=$(yum info installed pbench-agent 2> /dev/null | grep Version | awk '{ print $3 }')
 if [ -z "$ver" ]; then
     ver="unknown"
 fi
-rel=$(yum info installed pbench 2> /dev/null | grep Release | awk '{ print $3 }')
+rel=$(yum info installed pbench-agent 2> /dev/null | grep Release | awk '{ print $3 }')
 if [ -z "$ver" ]; then
     rel="unknown"
 fi
-# User-Agent HTTP header: <pbench-ver-rel>:<FQDN>:<$USER>:<full path of this script>""
-user_agent="pbench-$ver-$rel:$(hostname -f):$USER:$0"
+# User-Agent HTTP header: <pbench-agent-ver-rel>:<FQDN>:<$USER>:<full path of this script>""
+user_agent="pbench-agent-$ver-$rel:$(hostname -f):$USER:$0"
 
 results_host_info_url="http://$results_webserver/pbench-results-host-info.versioned/pbench-results-host-info.URL001"
 results_host_info=$(curl -s -A "$user_agent" -L "$results_host_info_url")

agent/util-scripts/register-tool

@@ -135,7 +135,7 @@ fi
 if [ -z "$remote" ]; then
 	if [ -e "$pbench_bin/tool-scripts/$name" ]; then
 		debug_log "checking to see if tool is installed..."
-		$pbench_bin/tool-scripts/$name --install
+		$pbench_bin/tool-scripts/$name --install "$tool_opts"
 		rc=$?
 	else
 		debug_log "Could not find $name in $pbench_bin/tool-scripts.  Has this tool been integrated in to pbench?"