diff --git a/CHANGELOG b/CHANGELOG index 957ec80c..dff26a53 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,37 @@ + COSBench 0.4.0.b2 (14-ww17.1) +------------------------------------ +. #57: time synchronization +. #96: Creating Web UI facility for generating different workload configuration files +. #155: Add support for "acceptable failure rate, failure %" to all stages +. #4: xml validator +. #77: contextual parameter list +. #167: storage policy supporting for openstack swift +. #37: support CDMI interface +. #139: NullPointerException when a mission is aborted + + + COSBench 0.4.0.b1 (14-ww11.4) +------------------------------------ +. #166: show driver alive state on controller index page +. #158: Extend "histogram" selector to support "open ranges" +. #165: enable Open range for object size in uniform and histogram +. #136: 1 GB Object Reads Fail due to IllegalArgumentException +. #99 (#152): measure and report the time for first and last byte in or out +. #159: Extend "histogram" selector to support "floating point" weights/percentages +. #143: at loading archived workloads, even there are 100 workloads, the UI shows only 98. +. #140: duplicated workload id generated +. #151: avoid being challenged for each request + + + COSBench 0.4.0.a1 (14-ww1.4) +------------------------------------ +. #137: when there are large # of archive jobs, exception will raise from archive list on web portal. +. #139: NullPointerException when a mission is aborted +. support http BASIC and DIGEST authentication +. add cdmi base and util bundle +. add cdmi adapter for swift+cdmi middleware + + COSBench 0.3.3.0 (13-ww44.5) ------------------------------------ . #20: multiple same operations @@ -56,81 +90,4 @@ COSBench 0.3.0.1 (13-ww17.5) COSBench 0.3.0.0 (13-ww15.5) ------------------------------------ -1. rebase version to 0.3.0 for open source - -====================================================================== - - COSBench 2.2.0.GA (13-ww15.1) ------------------------------------- -1. fix a bug related to socket connections -2. add a console option "-Dcosbench.controller.history" to help set the max number of workloads cosbench will hold in RAM -3. add a console option "-Dcosbench.driver.history" to help set the max number of missions cosbench will hold in RAM -4. add a config parameter "retry" to help user define the number of retries cosbench will attempt should previous efforts fail -5. improve log printing to help better debuging -6. miscellanous minor bug fixing - - COSBench 2.2.0.beta2 (13-ww08.2) ------------------------------------- -1. fix a bug which will cause OOM after a few days' intensive testing. -2. support ceph -3. support to forcily terminate load to avoid long tail due to high load. -4. add one parameter "tenant_name" for keystone configuration. -5. add retry logic to avoid one failed auth thread to terminate whole workload. -6. change parameters: - a) for swift, change parameter "url" to "auth_url"; - b) for keystone, add parameter "tenant_name". - - - COSBench 2.2.0.beta1 (12-ww48.5) ------------------------------------- -1. internal code refactoring -2. add one new bundle to wrap http related utilities and abstract Context class -3. formalize adaptor interfaces -4. provide one sample project for adaptor development -5. provide one development guide for adaptor - - COSBench 2.1.0.GA (12-ww44.5) ------------------------------------- -1. enhance start/stop scripts to handle unexpected cases. -2. add step by step installation manual in user guide document. -3. 2.1 release preparation: - - generate branch for 2.1 release preparation and switch to work on the branch for dev/release folder. - - change version to 2.1.0.GA - - remove version information in "README" -4. add storage line at dispose stage in ampli-config-sample.xml to support v2.5 - - COSBench 2.0.5.GA (12-ww43.4) ------------------------------------- -1. support object integrity check -2. finalize config editing page -3. unify sample config files - - COSBench 2.0.4.GA (12-ww42.3) ------------------------------------- -1. fix one bug which causes queuing workload can't be cancelled. -2. fix one bug which causes 100% read and 0% write operation mix can't work as expecteed. -3. unify mission and task on web page -4. draft version of config editing page - - COSBench 2.0.2.GA (12-ww38.3) ------------------------------------- -1. fix one bug in swift-config-sample.xml, which causes failure at dispose stage. -2. fix one bug for amplistor which can't get policy id defined in storage. -3. fix imbalanced work allocation between driver. -4. fix one bug which causes http connections not released after testing. - - - COSBench 2.0.1.GA (12-ww34.5) ------------------------------------- -1. enhance the content based on internal and external users� feedback. -2. make policy parameter optional for object related operation -3. support timeout for amplistor -4. fix a bug on "reset" button at matrix page - - COSBench 2.0.0.GA (12-ww31.5) ------------------------------------- -1. each workload stage now has a rampdown phase. -2. support latency histogram. -3. cosbench now has its own logging system. -4. resolved serval issues related to performance metrics calculation. -5. resolved serval issues related to HTTP client. +1. rebase version to 0.3.0 for open source \ No newline at end of file diff --git a/COSBench-User-Guide.odt b/COSBench-User-Guide.odt index 7f7075fe..6c310385 100644 Binary files a/COSBench-User-Guide.odt and b/COSBench-User-Guide.odt differ diff --git a/COSBenchUserGuide.pdf b/COSBenchUserGuide.pdf index e8f00ca4..b0f89528 100644 Binary files a/COSBenchUserGuide.pdf and b/COSBenchUserGuide.pdf differ diff --git a/dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java b/dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java index 89ae0f28..1749d99a 100644 --- a/dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java +++ b/dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java @@ -41,7 +41,7 @@ public class AmpliClient { private HttpClient client = null; /* current operation */ - private volatile HttpUriRequest request; + private volatile HttpUriRequest method; private int port; private String host; private String nsRoot; @@ -54,20 +54,20 @@ public AmpliClient(HttpClient client, String host, int port, String nsRoot) { } public void dispose() { - request = null; + method = null; HttpClientUtil.disposeHttpClient(client); } public void abort() { - if (request != null) - request.abort(); - request = null; + if (method != null) + method.abort(); + method = null; } public boolean login() throws IOException, HttpException { String storageUrl = "http://" + this.host + ":" + this.port; - HttpHead method = HttpClientUtil.makeHttpHead(storageUrl); + method = HttpClientUtil.makeHttpHead(storageUrl); HttpResponse response = null; try { response = client.execute(method); @@ -89,7 +89,6 @@ public String StoreObject(String sourceFilename, String ampliNamespace, AmpliException { File file = new File(sourceFilename); - HttpPut method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -97,7 +96,7 @@ public String StoreObject(String sourceFilename, String ampliNamespace, method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace) + "/" + HttpClientUtil.encodeURL(ampliFilename)); - method.setEntity(new FileEntity(file, "application/octet-stream")); + ((HttpPut)method).setEntity(new FileEntity(file, "application/octet-stream")); response = client.execute(method); @@ -127,7 +126,6 @@ public String StoreObject(String sourceFilename, String ampliNamespace, public String StoreStreamedObject(InputStream stream, long length, String ampliNamespace, String ampliFilename) throws IOException, HttpException, AmpliException { - HttpPut method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -143,7 +141,7 @@ public String StoreStreamedObject(InputStream stream, long length, entity.setChunked(false); } - method.setEntity(entity); + ((HttpPut)method).setEntity(entity); response = client.execute(method); @@ -191,7 +189,6 @@ public String StoreObject(byte[] data, String ampliNamespace, String ampliFilename) throws IOException, HttpException, AmpliException { // int len = data.length; - HttpPut method = null; String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace) + "/" + HttpClientUtil.encodeURL(ampliFilename)); @@ -200,7 +197,7 @@ public String StoreObject(byte[] data, String ampliNamespace, HttpResponse response = null; try { - method.setEntity(new ByteArrayEntity(data)); + ((HttpPut)method).setEntity(new ByteArrayEntity(data)); response = client.execute(method); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED) { return response.getFirstHeader("ETag").getValue(); @@ -222,7 +219,7 @@ public byte[] getObject(String namespace, String objName) throws IOException, HttpException, AmpliException { String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; - HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + "/" + HttpClientUtil.encodeURL(objName)); HttpResponse response = null; @@ -249,7 +246,7 @@ public InputStream getObjectAsStream(String namespace, String objName) throws IOException, HttpException, AmpliException { String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; - HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + "/" + HttpClientUtil.encodeURL(objName)); HttpResponse response = null; @@ -272,7 +269,6 @@ public InputStream getObjectAsStream(String namespace, String objName) public boolean deleteObject(String ampliNamespace, String name) throws HttpException, IOException, AmpliException { - HttpDelete method = null; HttpResponse response = null; try { @@ -302,7 +298,6 @@ public boolean deleteObject(String ampliNamespace, String name) public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException, IOException, AmpliException { - HttpPut method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -336,7 +331,6 @@ public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException, public AmpliPolicy getPolicy(String policyId) throws HttpException, IOException, AmpliException { - HttpGet method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -366,13 +360,13 @@ public AmpliPolicy getPolicy(String policyId) throws HttpException, public AmpliNamespace createNamespace(AmpliNamespace namespace) throws HttpException, IOException, AmpliException { - HttpPut method = null; - HttpResponse response = null; + + HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; - method = new HttpPut(storageUrl); + method = HttpClientUtil.makeHttpGet(storageUrl); method.setHeader("Content-Type", "text/plain"); @@ -405,7 +399,7 @@ public String createNamespace(String namespace, String policy_id) String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; - method = new HttpPut(storageUrl); + method = HttpClientUtil.makeHttpPut(storageUrl); method.setHeader("Content-Type", "text/plain"); @@ -438,7 +432,6 @@ public String createNamespace(String namespace, String policy_id) public AmpliNamespace getNamespace(String name) throws HttpException, IOException, AmpliException { - HttpGet method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -469,7 +462,6 @@ public AmpliNamespace getNamespace(String name) throws HttpException, public boolean isNamespaceExisted(String name) throws HttpException, IOException, AmpliException { - HttpHead method = null; HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -497,7 +489,7 @@ public boolean isNamespaceExisted(String name) throws HttpException, public boolean deleteNamespace(String namespace) throws HttpException, IOException, AmpliException { - HttpDelete method = null; + HttpResponse response = null; try { String storageUrl = "http://" + this.host + ":" + this.port @@ -525,7 +517,7 @@ public Map getObjectMetadata(String namespace, String objName) throws IOException, HttpException, AmpliException { String storageUrl = "http://" + this.host + ":" + this.port + nsRoot; - HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace) + "/" + HttpClientUtil.encodeURL(objName) + "?meta=http"); HttpResponse response = null; diff --git a/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF index a5dbe935..4d716fe5 100644 --- a/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF +++ b/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF @@ -8,20 +8,25 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, com.intel.cosbench.api.context, com.intel.cosbench.api.storage, + com.intel.cosbench.client.cdmi.util, com.intel.cosbench.client.http, com.intel.cosbench.config, com.intel.cosbench.log, - com.intel.cosbench.client.cdmi.util, org.apache.commons.codec;version="[1.3.0,2.0.0)", org.apache.commons.codec.binary;version="1.3.0", org.apache.commons.codec.net;version="[1.3.0,2.0.0)", org.apache.commons.io.input;version="[1.3.0,2.0.0)", org.apache.commons.lang.math;version="2.5.0", org.apache.http;version="[4.1.4,5.0.0)", + org.apache.http.auth.params;version="[4.1.3,5.0.0)", org.apache.http.client;version="[4.1.3,5.0.0)", org.apache.http.client.methods;version="[4.1.3,5.0.0)", + org.apache.http.client.params;version="[4.1.3,5.0.0)", + org.apache.http.client.protocol;version="[4.1.3,5.0.0)", org.apache.http.conn;version="[4.1.3,5.0.0)", org.apache.http.entity;version="[4.1.4,5.0.0)", + org.apache.http.impl.client;version="[4.1.3,5.0.0)", org.apache.http.message;version="[4.1.4,5.0.0)", org.apache.http.params;version="[4.1.4,5.0.0)", + org.apache.http.protocol;version="[4.1.4,5.0.0)", org.apache.http.util;version="[4.1.4,5.0.0)" diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/api/cdmi/base/CDMIStorage.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/api/cdmi/base/CDMIStorage.java index c6eb7da1..a77c928c 100644 --- a/dev/cosbench-cdmi-base/src/com/intel/cosbench/api/cdmi/base/CDMIStorage.java +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/api/cdmi/base/CDMIStorage.java @@ -23,11 +23,12 @@ public class CDMIStorage extends NoneStorage { // below parameters expect to get from configuration file. private int timeout; private String rootPath; + private String type; private String headers; private boolean flag; // local variables - private CdmiClient client; + private BaseCdmiClient client; private String[] header_list; @Override @@ -35,7 +36,7 @@ public void init(Config config, Logger logger) { super.init(config, logger); initParms(config); - client = new CdmiClient(false); + client = CdmiClientFactory.getClient(type); } private void initParms(Config config) { @@ -43,11 +44,13 @@ private void initParms(Config config) { timeout = config.getInt(TIMEOUT_KEY, TIMEOUT_DEFAULT); headers = config.get(CUSTOM_HEADERS_KEY, CUSTOM_HEADERS_DEFAULT); flag = config.getBoolean(RAISE_DELETE_ERRORS_KEY, RAISE_DELETE_ERRORS_DEFAULT); + type = config.get(CDMI_CONTENT_TYPE_KEY, CDMI_CONTENT_TYPE_DEFAULT); header_list = headers.split(","); parms.put(ROOT_PATH_KEY, rootPath); parms.put(TIMEOUT_KEY, timeout); parms.put(RAISE_DELETE_ERRORS_KEY, flag); + parms.put(CDMI_CONTENT_TYPE_KEY, type); } @Override @@ -70,7 +73,7 @@ public void setAuthContext(AuthContext info) { } logger.debug("httpclient =" + httpClient + ", url = " + url); - client.init(httpClient, url, headerKV); + client.init(httpClient, url, headerKV, false); } catch (Exception e) { throw new StorageException(e); } diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/BaseCdmiClient.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/BaseCdmiClient.java new file mode 100644 index 00000000..8b4930de --- /dev/null +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/BaseCdmiClient.java @@ -0,0 +1,51 @@ +package com.intel.cosbench.client.cdmi.base; + +import java.io.*; +import java.util.*; + +import org.apache.http.Header; +import org.apache.http.auth.params.AuthPNames; +import org.apache.http.client.*; +import org.apache.http.client.params.AuthPolicy; +import org.apache.http.client.protocol.ClientContext; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.*; + + +public abstract class BaseCdmiClient { + protected boolean raise_delete_errors = false; + protected HttpClient client; + protected HttpContext httpContext; + protected String uri; + protected ArrayList
custom_headers = new ArrayList
(); +// +// public BaseCdmiClient() { +// this.raise_delete_errors = flag; +// } + + public void init(HttpClient httpClient, String uri, Map headerKV, boolean flag) { + this.client = httpClient; + this.httpContext = new BasicHttpContext(); + httpContext.setAttribute(AuthPNames.TARGET_AUTH_PREF, Arrays.asList(new String[] {AuthPolicy.BASIC, AuthPolicy.DIGEST})); + + final AuthCache authCache = new BasicAuthCache(); + httpContext.setAttribute(ClientContext.AUTH_CACHE, authCache); + this.uri = uri; + this.raise_delete_errors = flag; + + for(String key: headerKV.keySet()) + this.custom_headers.add(new BasicHeader(key, headerKV.get(key))); + } + + public abstract void dispose(); + + public abstract void createContainer(String container) throws IOException, CdmiException; + public abstract void deleteContainer(String container) throws IOException, CdmiException; + public abstract InputStream getObjectAsStream(String container, String object) throws IOException, CdmiException; + + public abstract void storeStreamedObject(String container, String object, + InputStream data, long length) throws IOException, CdmiClientException; + public abstract void deleteObject(String container, String object) + throws IOException, CdmiException; +} diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClient.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClient.java index 191addb2..ea82bd13 100644 --- a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClient.java +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClient.java @@ -3,41 +3,46 @@ import static org.apache.http.HttpStatus.*; import java.io.*; -import java.util.ArrayList; -import java.util.Map; -import org.apache.http.client.HttpClient; import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.http.*; import org.apache.http.client.methods.*; import org.apache.http.entity.StringEntity; -import org.apache.http.message.BasicHeader; import org.apache.http.util.*; import com.intel.cosbench.client.cdmi.util.CdmiJsonInputStreamEntity; - -public class CdmiClient { - private boolean raise_delete_errors = false; - - private HttpClient client; - private String uri; - private ArrayList
custom_headers = new ArrayList
(); +/** + * This class encapsulates operations to access cdmi compatible server with cdmi content type. + * + * @author ywang19 + * + */ +public class CdmiClient extends BaseCdmiClient { +// private boolean raise_delete_errors = false; +// private HttpClient client; +// private String uri; +// private ArrayList
custom_headers = new ArrayList
(); private final static String cdmi_ver = "1.0.1"; - public CdmiClient(boolean flag) { - this.raise_delete_errors = flag; - } +// public CdmiClient(boolean flag) { +// this.raise_delete_errors = flag; +// } + + public CdmiClient() { + super(); + } - public void init(HttpClient httpClient, String uri, Map headerKV) { - this.client = httpClient; - this.uri = uri; - - for(String key: headerKV.keySet()) - this.custom_headers.add(new BasicHeader(key, headerKV.get(key))); - } +// public void init(HttpClient httpClient, String uri, Map headerKV, boolean flag) { +// this.client = httpClient; +// this.uri = uri; +// this.raise_delete_errors = flag; +// +// for(String key: headerKV.keySet()) +// this.custom_headers.add(new BasicHeader(key, headerKV.get(key))); +// } public void dispose() { client.getConnectionManager().shutdown(); @@ -62,7 +67,7 @@ public void createContainer(String container) throws IOException, method.setEntity(new StringEntity("{}")); - response = client.execute(method); + response = client.execute(method, httpContext); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == SC_CREATED || statusCode == SC_ACCEPTED) { @@ -87,7 +92,7 @@ public void deleteContainer(String container) throws IOException, method.setHeader("X-CDMI-Specification-Version", cdmi_ver); setCustomHeaders(method); - response = client.execute(method); + response = client.execute(method, httpContext); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == SC_NO_CONTENT) return; @@ -119,7 +124,7 @@ public InputStream getObjectAsStream(String container, String object) method.setHeader("X-CDMI-Specification-Version", cdmi_ver); setCustomHeaders(method); - response = client.execute(method); + response = client.execute(method, httpContext); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == SC_OK) return response.getEntity().getContent(); @@ -132,6 +137,7 @@ public InputStream getObjectAsStream(String container, String object) response.getAllHeaders(), response.getStatusLine()); } + @SuppressWarnings("unused") private void dumpMethod(HttpRequestBase method) { System.out.println("==== METHOD BEGIN ===="); System.out.println(method.getMethod()); @@ -142,6 +148,7 @@ private void dumpMethod(HttpRequestBase method) { System.out.println("==== METHOD END ===="); } + @SuppressWarnings("unused") private void dumpResponse(HttpResponse response) { System.out.println("==== RESPONSE BEGIN ===="); Header[] hdr = response.getAllHeaders(); @@ -184,7 +191,7 @@ public void storeStreamedObject(String container, String object, method.setEntity(entity); - response = client.execute(method); + response = client.execute(method, httpContext); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_CREATED) { return; @@ -215,7 +222,7 @@ public void deleteObject(String container, String object) method.setHeader("X-CDMI-Specification-Version", cdmi_ver); setCustomHeaders(method); - response = client.execute(method); + response = client.execute(method, httpContext); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == SC_NO_CONTENT) diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClientFactory.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClientFactory.java new file mode 100644 index 00000000..5789973d --- /dev/null +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiClientFactory.java @@ -0,0 +1,21 @@ +package com.intel.cosbench.client.cdmi.base; + +/** + * This class provides a method to create corresponding client to access cdmi compatible server in different form (like cdmi content type or non-cdmi content type). + * + * @author ywang19 + * + */ +public class CdmiClientFactory { + + public static BaseCdmiClient getClient(String type) { + if("cdmi".equalsIgnoreCase(type)) { // cdmi content type + return new CdmiClient(); + }else if("non-cdmi".equalsIgnoreCase(type)) { + return new NonCdmiClient(); + }else { + System.err.println("Type: " + type + " is not supported yet."); + return null; + } + } +} diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiConstants.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiConstants.java index b63942ff..1f45a8ed 100644 --- a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiConstants.java +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/CdmiConstants.java @@ -17,6 +17,9 @@ public interface CdmiConstants { String CUSTOM_HEADERS_KEY = "custom_headers"; String CUSTOM_HEADERS_DEFAULT = ""; + String CDMI_CONTENT_TYPE_KEY = "type"; + String CDMI_CONTENT_TYPE_DEFAULT = "cdmi"; + // -------------------------------------------------------------------------- // CONNECTION // -------------------------------------------------------------------------- diff --git a/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/NonCdmiClient.java b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/NonCdmiClient.java new file mode 100644 index 00000000..b1a249aa --- /dev/null +++ b/dev/cosbench-cdmi-base/src/com/intel/cosbench/client/cdmi/base/NonCdmiClient.java @@ -0,0 +1,230 @@ +package com.intel.cosbench.client.cdmi.base; + +import static org.apache.http.HttpStatus.*; + +import java.io.*; + +import org.apache.commons.codec.EncoderException; +import org.apache.commons.codec.net.URLCodec; +import org.apache.http.*; +import org.apache.http.client.methods.*; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.util.*; + + +/** + * This class encapsulates operations to access cdmi compatible server with non-cdmi content type. + * + * @author ywang19 + * + */ +public class NonCdmiClient extends BaseCdmiClient{ +// private HttpClient client; +// private String uri; +// private ArrayList
custom_headers = new ArrayList
(); + + public NonCdmiClient() { + super(); + } + +// public void init(HttpClient httpClient, String uri, Map headerKV) { +// this.client = httpClient; +// this.uri = uri; +// +// for(String key: headerKV.keySet()) +// this.custom_headers.add(new BasicHeader(key, headerKV.get(key))); +// } + + public void dispose() { + client.getConnectionManager().shutdown(); + } + + private void setCustomHeaders(HttpRequest method) { + for(Header header : custom_headers) + method.setHeader(header); + } + + public void createContainer(String container) throws IOException, + CdmiException { + HttpResponse response = null; + try { + // Create the request + HttpPut method = new HttpPut(uri + "/" + encodeURL(container) + "/"); + + setCustomHeaders(method); + + response = client.execute(method, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode == SC_CREATED || statusCode == SC_ACCEPTED) { + return; + } + throw new CdmiException("unexpected return from server", + response.getAllHeaders(), response.getStatusLine()); + }finally { + if (response != null) + EntityUtils.consume(response.getEntity()); + } + } + + public void deleteContainer(String container) throws IOException, + CdmiException { + // add storage access logic here. + HttpResponse response = null; + try { + // Create the request + HttpDelete method = new HttpDelete(uri + "/" + encodeURL(container) + "/"); // "http://localhost:8080/cdmi-server/TestContainer/"); + + setCustomHeaders(method); + + response = client.execute(method, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == SC_NO_CONTENT) + return; + if (statusCode == SC_NOT_FOUND) + throw new CdmiFileNotFoundException("container not found: " + + container, response.getAllHeaders(), + response.getStatusLine()); + if (statusCode == SC_CONFLICT) + throw new CdmiConflictException( + "cannot delete an non-empty container", + response.getAllHeaders(), response.getStatusLine()); + throw new CdmiException("unexpected return from server", + response.getAllHeaders(), response.getStatusLine()); + } finally { + if (response != null) + EntityUtils.consume(response.getEntity()); + } + } + + public InputStream getObjectAsStream(String container, String object) + throws IOException, CdmiException { + + HttpResponse response = null; + // Create the request + HttpGet method = new HttpGet(uri + "/" + encodeURL(container) + + "/" + encodeURL(object)); + + setCustomHeaders(method); + + response = client.execute(method, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == SC_OK) + return response.getEntity().getContent(); + + if (statusCode == SC_NOT_FOUND) + throw new CdmiFileNotFoundException("object not found: " + + container + "/" + object, response.getAllHeaders(), + response.getStatusLine()); + throw new CdmiException("unexpected result from server", + response.getAllHeaders(), response.getStatusLine()); + } + + @SuppressWarnings("unused") + private void dumpMethod(HttpRequestBase method) { + System.out.println("==== METHOD BEGIN ===="); + System.out.println(method.getMethod()); + System.out.println(method.getURI()); + for(Header header: method.getAllHeaders()) { + System.out.println(header.getName() + ": " + header.getValue()); + } + System.out.println("==== METHOD END ===="); + } + + @SuppressWarnings("unused") + private void dumpResponse(HttpResponse response) { + System.out.println("==== RESPONSE BEGIN ===="); + Header[] hdr = response.getAllHeaders(); + System.out.println("Headers : " + hdr.length); + for (int i = 0; i < hdr.length; i++) { + System.out.println(hdr[i]); + } + System.out.println("---------"); + System.out.println(response.getProtocolVersion()); + System.out.println(response.getStatusLine().getStatusCode()); + + System.out.println(response.getStatusLine().getReasonPhrase()); + System.out.println(response.getStatusLine().toString()); + System.out.println("---------"); + System.out.println("==== RESPONSE END ===="); + } + + public void storeStreamedObject(String container, String object, + InputStream data, long length) throws IOException, CdmiClientException { + // add storage access logic here. + HttpPut method = null; + // Create the request + HttpResponse response = null; + try { + method = new HttpPut(uri + "/" + encodeURL(container) + + "/" + encodeURL(object)); + + method.setHeader("Content-Type", "application/octet-stream"); + setCustomHeaders(method); + InputStreamEntity entity = new InputStreamEntity(data, length); + if (length < 0) + entity.setChunked(true); + else { + entity.setChunked(false); + } + + method.setEntity(entity); + + response = client.execute(method, httpContext); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_CREATED) { + return; + } if (statusCode == SC_ACCEPTED) + return; + if (statusCode == SC_NOT_FOUND) + throw new FileNotFoundException("container not found: " + + container); + else { + throw new CdmiClientException(statusCode, "Unexpected Server Response: " + + response.getStatusLine(), response.getAllHeaders(), + response.getStatusLine()); + } + }finally { + if (response != null) + EntityUtils.consume(response.getEntity()); + } + } + + public void deleteObject(String container, String object) + throws IOException, CdmiException { + HttpResponse response = null; + try { + // Create the request + HttpDelete method = new HttpDelete(uri + "/" + encodeURL(container) + + "/" + encodeURL(object)); + + setCustomHeaders(method); + + response = client.execute(method, httpContext); + + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == SC_NO_CONTENT) + return; + if (!raise_delete_errors) + return; + if (statusCode == SC_NOT_FOUND) + throw new CdmiFileNotFoundException("object not found: " + + container + "/" + object, + response.getAllHeaders(), response.getStatusLine()); + throw new CdmiException("unexpected return from server", + response.getAllHeaders(), response.getStatusLine()); + } finally { + if (response != null) + EntityUtils.consume(response.getEntity()); + } + } + + public static String encodeURL(String str) { + URLCodec codec = new URLCodec(); + try { + return codec.encode(str).replaceAll("\\+", "%20"); + } catch (EncoderException ee) { + return str; + } + } +} diff --git a/dev/cosbench-cdmi-swift/src/com/intel/cosbench/client/cdmiswift/CdmiSwiftClient.java b/dev/cosbench-cdmi-swift/src/com/intel/cosbench/client/cdmiswift/CdmiSwiftClient.java index 159315d1..396ca0e8 100644 --- a/dev/cosbench-cdmi-swift/src/com/intel/cosbench/client/cdmiswift/CdmiSwiftClient.java +++ b/dev/cosbench-cdmi-swift/src/com/intel/cosbench/client/cdmiswift/CdmiSwiftClient.java @@ -124,6 +124,7 @@ public InputStream getObjectAsStream(String container, String object) response.getAllHeaders(), response.getStatusLine()); } + @SuppressWarnings("unused") private void dumpMethod(HttpRequestBase method) { System.out.println("==== METHOD BEGIN ===="); System.out.println(method.getMethod()); @@ -134,6 +135,7 @@ private void dumpMethod(HttpRequestBase method) { System.out.println("==== METHOD END ===="); } + @SuppressWarnings("unused") private void dumpResponse(HttpResponse response) { System.out.println("==== RESPONSE BEGIN ===="); Header[] hdr = response.getAllHeaders(); diff --git a/dev/cosbench-cdmi-util/src/com/intel/cosbench/client/cdmi/util/CdmiJsonInputStreamEntity.java b/dev/cosbench-cdmi-util/src/com/intel/cosbench/client/cdmi/util/CdmiJsonInputStreamEntity.java index cfdf019f..bef9ea64 100644 --- a/dev/cosbench-cdmi-util/src/com/intel/cosbench/client/cdmi/util/CdmiJsonInputStreamEntity.java +++ b/dev/cosbench-cdmi-util/src/com/intel/cosbench/client/cdmi/util/CdmiJsonInputStreamEntity.java @@ -15,9 +15,9 @@ import org.apache.http.message.BasicHeader; /** - * This class encapsulates an inputstream which can handle the json structure adopted by cdmi content type. + * This class encapsulates an input stream which can handle the json structure adopted by cdmi content type. * - * @author ywang19, qzheng7 + * @author ywang19 * */ public class CdmiJsonInputStreamEntity extends AbstractHttpEntity diff --git a/dev/cosbench-config/bin/com/intel/cosbench/config/castor/work-mapping.xml b/dev/cosbench-config/bin/com/intel/cosbench/config/castor/work-mapping.xml index d085affe..48efc4c2 100644 --- a/dev/cosbench-config/bin/com/intel/cosbench/config/castor/work-mapping.xml +++ b/dev/cosbench-config/bin/com/intel/cosbench/config/castor/work-mapping.xml @@ -37,6 +37,10 @@ + + + + @@ -45,7 +49,7 @@ - + diff --git a/dev/cosbench-config/src/com/intel/cosbench/config/ConfigConstants.java b/dev/cosbench-config/src/com/intel/cosbench/config/ConfigConstants.java new file mode 100644 index 00000000..177c0808 --- /dev/null +++ b/dev/cosbench-config/src/com/intel/cosbench/config/ConfigConstants.java @@ -0,0 +1,7 @@ +package com.intel.cosbench.config; + +public interface ConfigConstants { + + String DELIMITER=","; + +} diff --git a/dev/cosbench-config/src/com/intel/cosbench/config/Stage.java b/dev/cosbench-config/src/com/intel/cosbench/config/Stage.java index d20a9139..c6ab71ec 100644 --- a/dev/cosbench-config/src/com/intel/cosbench/config/Stage.java +++ b/dev/cosbench-config/src/com/intel/cosbench/config/Stage.java @@ -51,6 +51,8 @@ public String getName() { public void setName(String name) { if (StringUtils.isEmpty(name)) throw new ConfigException("stage name cannot be empty"); + if (StringUtils.containsAny(name, ConfigConstants.DELIMITER)) + throw new ConfigException("stage name cannot contain delimiter '" + ConfigConstants.DELIMITER + "'"); this.name = name; } diff --git a/dev/cosbench-config/src/com/intel/cosbench/config/Work.java b/dev/cosbench-config/src/com/intel/cosbench/config/Work.java index d3b2a36a..5b254abf 100644 --- a/dev/cosbench-config/src/com/intel/cosbench/config/Work.java +++ b/dev/cosbench-config/src/com/intel/cosbench/config/Work.java @@ -39,12 +39,14 @@ public class Work implements Iterable { private int runtime = 0; private int rampup = 0; private int rampdown = 0; + private int afr = 200000; /* acceptable failure ratio, the unit is samples per one million, + * default is 200000 for normal work, and 0 for init/prepare/cleanup/dispose/delay work */ private int totalOps = 0; private long totalBytes = 0; private String driver; private String config; private Auth auth; - private Storage storage; + private Storage storage; private List operations; public Work() { @@ -204,6 +206,25 @@ public void setStorage(Storage storage) { throw new ConfigException("a work must have its storge"); this.storage = storage; } + + public int getAfr() { + return afr; + } + + public void setAfr(int afr) { + if (afr > 1000000 || afr < 0) + throw new ConfigException("afr should be at 0 to 1000000 range"); + this.afr = afr; + } + + public List getOperationIDs() { + List opIds = new ArrayList(); + for (Operation operation : operations) { + opIds.add(operation.getId()); + } + return opIds; + } + public List getOperations() { return operations; @@ -233,6 +254,7 @@ private void toPrepareWork() { name = "prepare"; setDivision("object"); setRuntime(0); + setAfr(0); setTotalBytes(0); setTotalOps(getWorkers()); Operation op = new Operation(); @@ -252,6 +274,7 @@ private void toCleanupWork() { name = "cleanup"; setDivision("object"); setRuntime(0); + setAfr(0); setTotalBytes(0); setTotalOps(getWorkers()); Operation op = new Operation(); @@ -271,6 +294,7 @@ private void toInitWork() { name = "init"; setDivision("container"); setRuntime(0); + setAfr(0); setTotalBytes(0); setTotalOps(getWorkers()); Operation op = new Operation(); @@ -286,6 +310,7 @@ private void toDisposeWork() { name = "dispose"; setDivision("container"); setRuntime(0); + setAfr(0); setTotalBytes(0); setTotalOps(getWorkers()); Operation op = new Operation(); @@ -301,6 +326,7 @@ public void toDelayWork() { name = "delay"; setDivision("none"); setRuntime(0); + setAfr(0); setTotalBytes(0); setWorkers(1); setTotalOps(getWorkers()); diff --git a/dev/cosbench-config/src/com/intel/cosbench/config/castor/work-mapping.xml b/dev/cosbench-config/src/com/intel/cosbench/config/castor/work-mapping.xml index d085affe..48efc4c2 100644 --- a/dev/cosbench-config/src/com/intel/cosbench/config/castor/work-mapping.xml +++ b/dev/cosbench-config/src/com/intel/cosbench/config/castor/work-mapping.xml @@ -37,6 +37,10 @@ + + + + @@ -45,7 +49,7 @@ - + diff --git a/dev/cosbench-controller-web/WEB-INF/freemarker/advanced-config.ftl b/dev/cosbench-controller-web/WEB-INF/freemarker/advanced-config.ftl index ab50cdac..6704a04a 100644 --- a/dev/cosbench-controller-web/WEB-INF/freemarker/advanced-config.ftl +++ b/dev/cosbench-controller-web/WEB-INF/freemarker/advanced-config.ftl @@ -11,7 +11,9 @@

Workload Matrix Configuration

-
(You can configure workload matrix from here. You can also create and submit generated workload configs.)
+
You can configure workload matrix from here. You can generate or submit workloads directly. With 'Generate Workload File/s' option, + files will be generated and placed at 'workloads' directory inside COSBench installation directory. No workload files will be generated when + 'Submit Workload/s' option is chosen and workloads will be submitted directly.
<#if error?? > @@ -104,6 +106,7 @@
+ @@ -202,6 +205,7 @@ \ No newline at end of file diff --git a/dev/cosbench-controller-web/WEB-INF/freemarker/config.ftl b/dev/cosbench-controller-web/WEB-INF/freemarker/config.ftl index bd938a36..416b1b5a 100755 --- a/dev/cosbench-controller-web/WEB-INF/freemarker/config.ftl +++ b/dev/cosbench-controller-web/WEB-INF/freemarker/config.ftl @@ -58,32 +58,35 @@ @@ -538,6 +541,83 @@ numOfClonesInStage[stageNum]++; previousDivs[stageNum] = cloneDiv; } + + function changeAuth() + { + var select=document.getElementById("auth.type"); + var selected=select.options[select.selectedIndex].value; + var config=document.getElementById("auth.config"); + + switch(selected) + { + case "swauth": + config.value="username=;password=;auth_url="; + config.title="e.g., username=test:tester;password=testing;auth_url=http://192.168.0.1:8080/auth/v1.0"; + break; + case "keystone": + config.value="username=;password=;tenant_name=;auth_url=;service="; + config.title="e.g., username=tester;password=testing;tenant_name=test;auth_url=http://127.0.0.1:5000/v2.0;service=swift service"; + break; + case "httpauth": + config.value="username=;password=;auth_url="; + config.title="e.g., username=tester;password=testing;auth_url=http://192.168.10.1:8080/cdmi"; + break; + case "mock": + config.value="delay= + <#list cInfo.driverInfos as dInfo > @@ -103,6 +104,11 @@ + <#if dInfo.aliveState> + + <#else> + + diff --git a/dev/cosbench-controller-web/WEB-INF/freemarker/matrix.ftl b/dev/cosbench-controller-web/WEB-INF/freemarker/matrix.ftl index 42b70448..06c2cec9 100644 --- a/dev/cosbench-controller-web/WEB-INF/freemarker/matrix.ftl +++ b/dev/cosbench-controller-web/WEB-INF/freemarker/matrix.ftl @@ -43,6 +43,8 @@ <#if allMetrics!false || bc!false >checked="true" /> Byte Count checked="true" /> Avg ResTime + checked="true" /> Avg ProcTime checked="true" /> Throughput <#if _99rt!false > <#if _100rt!false > + <#if allMetrics!false || pt!false > <#if allMetrics!false || t!false > <#if allMetrics!false || bw!false > <#if allMetrics!false || succ!false > @@ -223,6 +226,16 @@ + <#if allMetrics!false || pt!false > + + <#if allMetrics!false || t!false > diff --git a/dev/cosbench-controller-web/WEB-INF/freemarker/metrics.ftl b/dev/cosbench-controller-web/WEB-INF/freemarker/metrics.ftl index 2f24ff02..3c66caff 100644 --- a/dev/cosbench-controller-web/WEB-INF/freemarker/metrics.ftl +++ b/dev/cosbench-controller-web/WEB-INF/freemarker/metrics.ftl @@ -5,6 +5,7 @@ + @@ -67,6 +68,14 @@ ${mInfo.avgResTime?string("0.##")} ms + + @@ -101,6 +102,15 @@ ${mInfo.avgResTime?string("0.##")} ms + <#list allMetrics as mInfo > + + + <#list allMetrics as mInfo > diff --git a/dev/cosbench-controller-web/resources/cosbench.css b/dev/cosbench-controller-web/resources/cosbench.css index 90fcad6f..24b750f1 100644 --- a/dev/cosbench-controller-web/resources/cosbench.css +++ b/dev/cosbench-controller-web/resources/cosbench.css @@ -263,6 +263,28 @@ a:visited { /* --------------------------------------- * State Styles * --------------------------------------- */ +.alive { + width: 16px; + height: 16px; + background: #33CC00; + -moz-border-radius: 8px; + -webkit-border-radius: 8px; + border-radius: 8px; + margin:0px auto; + text-align:center; +} + +.dead { + width: 16px; + height: 16px; + background: #FF0000; + -moz-border-radius: 8px; + -webkit-border-radius: 8px; + border-radius: 8px; + margin:0px auto; + text-align:center; +} + .state { padding: 2px; color: #FFFFFF; diff --git a/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadConfigGenerator.java b/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadConfigGenerator.java index 28d3b848..da77a713 100644 --- a/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadConfigGenerator.java +++ b/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadConfigGenerator.java @@ -33,6 +33,7 @@ public class WorkloadConfigGenerator { private String storage_config; private boolean generateWorkloadFiles; private File WORKLOAD_CONFIG_DIR; + private static final String workloadConfigFilesRoot = "workloads"; protected ControllerService controller; private static final Logger LOGGER = LogFactory.getSystemLogger(); @@ -43,29 +44,32 @@ public WorkloadConfigGenerator(ControllerService controller) { } - public void createWorkloadFiles(HttpServletRequest req) { + public void createWorkloadFiles(HttpServletRequest req) throws Exception { // Set common workload parameters setWorkloadParams(req); String workloadMatrixName = req.getParameter("workload.matrix.name"); - if(workloadMatrixName == null) - workloadMatrixName = "workload-configs"; - WORKLOAD_CONFIG_DIR = new File(workloadMatrixName); - if (!WORKLOAD_CONFIG_DIR.exists()) - WORKLOAD_CONFIG_DIR.mkdirs(); - String path = WORKLOAD_CONFIG_DIR.getAbsolutePath(); - LOGGER.info("using {} for storing generated workload configs", path); - + if(!workloadMatrixName.matches("[a-zA-Z0-9\\_\\-#\\.\\(\\)\\/%&]{3,50}")) + throw new Exception("Workload Matrix Name incorrect. Please use alphabets or numbers. Special characters allowed are _ - # . ( ) / % &. " + + "Length should be between 3 to 50 characters."); String objectSizeStrings[] = req.getParameterValues("object-sizes"); - if (objectSizeStrings == null) return; - + for (int i = 0; i < objectSizeStrings.length; i++) { + String workloadName = req.getParameterValues("workload.name")[i]; + if(!workloadName.matches("[a-zA-Z0-9\\_\\-#\\.\\(\\)\\/%&]{3,50}")) + throw new Exception("Workload Name incorrect. Please use alphabets or numbers. Special characters allowed are _ - # . ( ) / % &. " + + "Length should be between 3 to 50 characters."); + } + + String workloadNumbers[] = req.getParameterValues("workload-number"); + for (int i = 0; i < workloadNumbers.length; i++) { String objectSizes[], unit; boolean isRange; - + + int workloadNumber = Integer.parseInt(workloadNumbers[i]); String objectSizeString = objectSizeStrings[i]; // if input is range of object sizes if (objectSizeString.contains("-")) { @@ -88,7 +92,8 @@ public void createWorkloadFiles(HttpServletRequest req) { .split(","); //Get read, write and delete ratios for one workload in string array. - String rWDRatios[] = getRWDRatios(req,i); + + String rWDRatios[] = getRWDRatios(req,workloadNumber+""); // parsing comma separated worker values String workers[] = req.getParameterValues("workers")[i].split(","); @@ -99,21 +104,27 @@ public void createWorkloadFiles(HttpServletRequest req) { workload.validate(); - String workloadName = req.getParameterValues("workload.name")[i]; - if(workloadName == null) - workloadName = "objSizes-"+objectSizeString+unit; + String workloadName = req.getParameterValues("workload.name")[workloadNumber]; + workload.setName(workloadName); if (generateWorkloadFiles) { + WORKLOAD_CONFIG_DIR = new File(workloadConfigFilesRoot+"/"+workloadMatrixName); + if (!WORKLOAD_CONFIG_DIR.exists()) + WORKLOAD_CONFIG_DIR.mkdirs(); + String path = WORKLOAD_CONFIG_DIR.getAbsolutePath(); + LOGGER.info("using {} for storing generated workload configs", path); + printWorkloadConfigXML(workload, workloadName); } else + { submitWorkload(workload); - + } } } - private String[] getRWDRatios(HttpServletRequest req, int workloadNumber) { + private String[] getRWDRatios(HttpServletRequest req, String workloadNumber) { String[] readRatios = req.getParameterValues("read-ratio"+workloadNumber); String[] writeRatios = req.getParameterValues("write-ratio"+workloadNumber); String[] deleteRatios = req.getParameterValues("delete-ratio"+workloadNumber); @@ -137,10 +148,7 @@ private void printWorkloadConfigXML(Workload workload, String workloadName) { try { String workloadXml = CastorConfigTools.getWorkloadWriter() .toXmlString(workload); - File singleWorkloadDirectory = new File(WORKLOAD_CONFIG_DIR+"/"+workloadName); - if (!singleWorkloadDirectory.exists()) - singleWorkloadDirectory.mkdirs(); - PrintWriter out = new PrintWriter(new File(singleWorkloadDirectory,workloadName+ ".xml")); + PrintWriter out = new PrintWriter(new File(WORKLOAD_CONFIG_DIR+"/"+workloadName+ ".xml")); out.print(workloadXml); out.close(); } catch (FileNotFoundException e) { diff --git a/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadMatrixConfigurationController.java b/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadMatrixConfigurationController.java index e1c42399..2691072e 100644 --- a/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadMatrixConfigurationController.java +++ b/dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadMatrixConfigurationController.java @@ -28,14 +28,13 @@ protected ModelAndView process(HttpServletRequest req, constructWorkloadConfigsFromPostData(req); } catch (Exception e) { - e.printStackTrace(); return createErrResult(e.getMessage()); } return createSuccResult(); } - private void constructWorkloadConfigsFromPostData(HttpServletRequest req) { + private void constructWorkloadConfigsFromPostData(HttpServletRequest req) throws Exception { WorkloadConfigGenerator wlConfGen = new WorkloadConfigGenerator(controller); wlConfGen.createWorkloadFiles(req); diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java index c2461bf8..2a0a1651 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java @@ -69,13 +69,12 @@ public synchronized void archive(WorkloadInfo info) { } catch (Exception e) { LOGGER.error("fail to archive workload", e); return; - } finally { - try { - updateCount(); - } catch (Exception e) { - LOGGER.error("cannot update workload count", e); - } } + try { + updateCount(info); + } catch (Exception e) { + LOGGER.error("fail to update count", e); + } String id = info.getId(); LOGGER.info("workload {} has been successfully archived", id); } @@ -240,21 +239,21 @@ private void exportPerformanceMatrix(WorkloadInfo info) throws IOException { String msg = "perf details of workload {} has been added to {}"; LOGGER.debug(msg, id, path); } - - private void updateCount() throws IOException { + + private void updateCount(WorkloadInfo info) throws IOException { int count = 0; File file = new File(ARCHIVE_DIR, ".meta"); - if (file.exists() && file.length() > 0) { - Reader reader = new BufferedReader(new FileReader(file)); - try { - count = new Scanner(reader).nextInt(); - } finally { - reader.close(); - } - } + String workloadId = null; + workloadId = info.getId(); + try { + count = parseID(workloadId); + } catch (Exception e) { + LOGGER.error("cannot parse workloadId", e); + } + Writer writer = new BufferedWriter(new FileWriter(file)); try { - writer.write(String.valueOf(++count)); + writer.write(String.valueOf(count)); } finally { writer.close(); } @@ -286,5 +285,15 @@ private int retrieveCount() throws IOException { LOGGER.debug("workload count has been retrieved as {}", count); return count; } + + private int parseID(String workloadID) throws NumberFormatException, IndexOutOfBoundsException { + int count = 0; + try { + count = Integer.parseInt(workloadID.substring(1)); + } catch (Exception e) { + LOGGER.error("cannot extract count from workloadId", e); + } + return count; + } } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVSnapshotLoader.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVSnapshotLoader.java index d5cf5a61..23e94693 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVSnapshotLoader.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVSnapshotLoader.java @@ -71,9 +71,9 @@ protected void readSnapshot() throws IOException { e.printStackTrace(); } Snapshot snapshot = new Snapshot(loadReport(columns), timestamp); - snapshot.setMinVersion(Integer.valueOf(columns[1 + opNum * 6])); - snapshot.setVersion(Integer.valueOf(columns[2 + opNum * 6])); - snapshot.setMaxVersion(Integer.valueOf(columns[3 + opNum * 6])); + snapshot.setMinVersion(Integer.valueOf(columns[1 + opNum * 7])); + snapshot.setVersion(Integer.valueOf(columns[2 + opNum * 7])); + snapshot.setMaxVersion(Integer.valueOf(columns[3 + opNum * 7])); stageContext.getSnapshotRegistry().addItem(snapshot); } } @@ -101,13 +101,16 @@ private List loadMetrics(String[] columns) { } metric.setSampleCount(getIntValue(columns[i + 1])); metric.setByteCount(getLongValue(columns[i + opNum + 1])); - metric.setAvgResTime(getDoubleValue(columns[i + opNum * 2 + 1])); - metric.setThroughput(getDoubleValue(columns[i + opNum * 3 + 1])); - metric.setBandwidth(getDoubleValue(columns[i + opNum * 4 + 1])); + double rt = getDoubleValue(columns[i + opNum * 2 + 1]); + metric.setAvgResTime(rt); + double pt = getDoubleValue(columns[i + opNum * 3 + 1]); + metric.setAvgXferTime(rt - pt); + metric.setThroughput(getDoubleValue(columns[i + opNum * 4 + 1])); + metric.setBandwidth(getDoubleValue(columns[i + opNum * 5 + 1])); // metric.setRatio(columns[i + opNum * 5 + 1].equalsIgnoreCase("N/A") ? 0D // : Double.valueOf(columns[i + opNum * 5 + 1].substring(0, // columns[i + opNum * 5 + 1].length() - 1)) / 100.0); - setRatio(columns[i + opNum * 5 + 1], metric); + setRatio(columns[i + opNum * 6 + 1], metric); metrics.add(metric); } return metrics; diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVWorkloadFileLoader.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVWorkloadFileLoader.java index 83818ef3..c2a74531 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVWorkloadFileLoader.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/loader/CSVWorkloadFileLoader.java @@ -57,7 +57,7 @@ protected void readWorkload() throws IOException { stageId = "s" + index++; sameStage = false; } - if (columns[15].equalsIgnoreCase("completed")) { + if (columns[16].equalsIgnoreCase("completed")) { Metrics metrics = loadMetrics(columns); if (!sameStage) { Report report = new Report(); @@ -68,13 +68,13 @@ protected void readWorkload() throws IOException { workloadContext.getReport().addMetrics(metrics); } for (StageState state : StageState.values()) { - if (columns[15] + if (columns[16] .equalsIgnoreCase(state.toString().toLowerCase())) { workloadContext.getStageInfo(stageId).setState(state, true); break; } } - int pos = 15; + int pos = 16; while (!sameStage && ++pos <= columns.length - 1) { String str[] = columns[pos].split("@"); String stateName = str[0].trim(); @@ -104,12 +104,14 @@ private Metrics loadMetrics(String[] columns) { metrics.setOpType(columns[2]); metrics.setSampleCount(Integer.valueOf(columns[3])); metrics.setByteCount(Long.valueOf(columns[4])); - metrics.setAvgResTime(columns[5].equalsIgnoreCase("N/A") ? 0 : Double - .valueOf(columns[5])); + double rt = columns[5].equalsIgnoreCase("N/A") ? 0 : Double.valueOf(columns[5]); + metrics.setAvgResTime(rt); + double pt = columns[6].equalsIgnoreCase("N/A") ? 0 : Double.valueOf(columns[6]); + metrics.setAvgXferTime(rt - pt); metrics.setLatency(loadHistogram(columns)); - metrics.setThroughput(Double.valueOf(columns[12])); - metrics.setBandwidth(Double.valueOf(columns[13])); - setRatio(columns[14], metrics); + metrics.setThroughput(Double.valueOf(columns[13])); + metrics.setBandwidth(Double.valueOf(columns[14])); + setRatio(columns[15], metrics); return metrics; } @@ -129,28 +131,28 @@ private void setRatio(String column, Metrics metrics) { private Histogram loadHistogram(String[] columns) { Histogram histogram = new Histogram(); long[] l_60 = new long[2]; - l_60[1] = columns[6].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[6]); + l_60[1] = columns[7].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[7]); histogram.set_60(l_60); long[] l_80 = new long[2]; - l_80[1] = columns[7].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[7]); + l_80[1] = columns[8].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[8]); histogram.set_80(l_80); long[] l_90 = new long[2]; - l_90[1] = columns[8].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[8]); + l_90[1] = columns[9].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[9]); histogram.set_90(l_90); long[] l_95 = new long[2]; - l_95[1] = columns[9].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[9]); + l_95[1] = columns[10].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[10]); histogram.set_95(l_95); long[] l_99 = new long[2]; - l_99[1] = columns[10].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[10]); + l_99[1] = columns[11].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[11]); histogram.set_99(l_99); long[] l_100 = new long[2]; - l_100[1] = columns[11].equalsIgnoreCase("N/A") ? 0L : Long - .valueOf(columns[11]); + l_100[1] = columns[12].equalsIgnoreCase("N/A") ? 0L : Long + .valueOf(columns[12]); histogram.set_100(l_100); return histogram; } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/DriverContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/DriverContext.java index 834b5232..e518a325 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/DriverContext.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/DriverContext.java @@ -30,6 +30,7 @@ public class DriverContext implements DriverInfo, MapRegistry.Item { private String name; private String url; + private boolean aliveState; public DriverContext() { /* empty */ @@ -53,4 +54,14 @@ public void setUrl(String url) { this.url = url; } + @Override + public void setAliveState(boolean aliveState) { + this.aliveState = aliveState; + } + + @Override + public boolean getAliveState(){ + return aliveState; + } + } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java index 78701166..c081c7a4 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java @@ -89,7 +89,7 @@ private void fireStageStopped() { listener.stageStopped(this); } - private Report mergeReport() { + public Report mergeReport() { if (taskRegistry == null) return new Report(); ReportMerger merger = new ReportMerger(); diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/TaskContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/TaskContext.java index 60def7d4..c7c359b5 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/TaskContext.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/TaskContext.java @@ -129,11 +129,13 @@ public void setLog(String log) { @Override public void disposeRuntime() { - httpClient = null; - mapper = null; - report = null; - log = null; - snapshot = new Snapshot(); + if(TaskState.isStopped(state)) { + httpClient = null; + mapper = null; + report = null; + log = null; + snapshot = new Snapshot(); + } } } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/SimpleWorkloadList.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/SimpleWorkloadList.java index 995f8ef9..ccb8aa0c 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/SimpleWorkloadList.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/SimpleWorkloadList.java @@ -51,7 +51,7 @@ public WorkloadContext fetch(String id) { public WorkloadContext[] add(WorkloadContext workload) { toBeRemoved.clear(); // begin transaction list.put(workload.getId(), workload); - capacity += 1; + capacity = count(); // size of list shrinkListSize(); WorkloadContext[] result = new WorkloadContext[toBeRemoved.size()]; result = toBeRemoved.toArray(result); diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerService.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerService.java index 10001a25..8d0944e5 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerService.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerService.java @@ -58,7 +58,6 @@ class COSBControllerService implements ControllerService, WorkloadListener { private WorkloadRepository memRepo = new RAMWorkloadRepository(); private boolean loadArch = false; - private boolean loaded = false; public COSBControllerService() { /* empty */ @@ -66,6 +65,11 @@ public COSBControllerService() { public void setContext(ControllerContext context) { this.context = context; + + // ping drivers and set alive state + Thread pingDriverThread = new Thread( + new PingDriverRunner(context.getDriverInfos())); + pingDriverThread.start(); } public void init() { @@ -88,6 +92,7 @@ public void init() { TimeUnit.MILLISECONDS, new PriorityBlockingQueue( memRepo.getMaxCapacity(), new OrderFutureComparator())); + } public void loadArchivedWorkload() throws IOException { @@ -96,7 +101,6 @@ public void loadArchivedWorkload() throws IOException { return; for (WorkloadInfo workloadContext : workloadContexts) memRepo.saveWorkload((WorkloadContext) workloadContext); - loaded = true; } public void unloadArchivedWorkload() { @@ -104,7 +108,6 @@ public void unloadArchivedWorkload() { memRepo.removeWorkload(workload); workload = null; } - loaded = false; } @@ -153,12 +156,15 @@ public boolean getloadArch() { public void setloadArch(boolean loadArch) { this.loadArch = loadArch; - if(getloadArch() && !loaded) + if(getloadArch()){ try { loadArchivedWorkload(); } catch (IOException e) { e.printStackTrace(); } + } else { + unloadArchivedWorkload(); + } } private String generateWorkloadId() { diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerServiceFactory.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerServiceFactory.java index eb52a50b..06c320bd 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerServiceFactory.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/COSBControllerServiceFactory.java @@ -117,6 +117,7 @@ private DriverContext getDriverContext(int index) { DriverContext context = new DriverContext(); context.setName(loadDriverName(index)); context.setUrl(loadDriverUrl(index)); + context.setAliveState(false); return context; } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/PingDriverRunner.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/PingDriverRunner.java new file mode 100644 index 00000000..6c2cd8cc --- /dev/null +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/PingDriverRunner.java @@ -0,0 +1,66 @@ +/** + +Copyright 2013 Intel Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package com.intel.cosbench.controller.service; + +import java.net.InetAddress; + +import com.intel.cosbench.model.DriverInfo; + +public class PingDriverRunner implements Runnable{ + + private int interval = 5000; + private DriverInfo[] driverInfos; + + PingDriverRunner(DriverInfo[] driverInfos){ + this.driverInfos = driverInfos; + } + + @Override + public void run() { + while (true) { + pingDrivers(driverInfos); + try { + Thread.sleep(interval); + } catch (InterruptedException ignore) { + } + } + } + + private void pingDrivers(DriverInfo[] driverInfos) { + for (DriverInfo driver : driverInfos) { + boolean isAlive = false; + + String ipAddress = getIpAddres(driver.getUrl()); + try { + if (!ipAddress.isEmpty()) { + isAlive = InetAddress.getByName(ipAddress).isReachable(3000); + } + } catch (Exception ignore) { + } + driver.setAliveState(isAlive); + } + } + + private String getIpAddres(String url) { + int start = url.indexOf('/') + 2; + int end = url.lastIndexOf(':'); + return end > start ? url.substring(start, end) : null; + } + +} + diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/StageRunner.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/StageRunner.java index 64a7feeb..67fcffe7 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/StageRunner.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/StageRunner.java @@ -22,7 +22,9 @@ import java.util.*; import java.util.concurrent.*; +import com.intel.cosbench.bench.Metrics; import com.intel.cosbench.config.Stage; +import com.intel.cosbench.config.Work; import com.intel.cosbench.controller.model.*; import com.intel.cosbench.controller.schedule.*; import com.intel.cosbench.controller.tasklet.*; @@ -144,8 +146,42 @@ private void runStage() { return; } } + + if (!reachAFRGoal()) { + stageContext.setState(FAILED); + return; + } stageContext.setState(COMPLETED); } + + private boolean reachAFRGoal() { + String id = stageContext.getId(); + boolean bool = true; + stageContext.setReport(stageContext.mergeReport()); + for (Work work : stageContext.getStage().getWorks()) { + List operationIDs = work.getOperationIDs(); + long sumSampleCount = 0; + long sumTotalSampleCount = 0; + for (Metrics metric : stageContext.getReport().getAllMetrics()) { + if (operationIDs.contains(metric.getOpId())) { + sumSampleCount += + metric.getSampleCount() > 0 ? metric.getSampleCount() : 0; + sumTotalSampleCount += + metric.getTotalSampleCount() > 0 ? metric.getTotalSampleCount() : 0; + } + } + LOGGER.info("acceptable failure ratio of work {} = {}", id+"-"+work.getName(), (double)work.getAfr() / 1000000); + LOGGER.info("real failure ratio of work {} = {}", id+"-"+work.getName(), + sumTotalSampleCount > 0 ? (double)(sumTotalSampleCount - sumSampleCount) / sumTotalSampleCount : "N/A"); + if ((sumTotalSampleCount - sumSampleCount) > sumTotalSampleCount * work.getAfr() / 1000000) { + LOGGER.info("fail to reach the goal of acceptable failure ratio in stage {} - work {}", id, work.getName()); + bool = false; + continue; + } + LOGGER.info("successfully reach the goal of acceptable failure ratio in stage {} - work {}", id, work.getName()); + } + return bool; + } private void bootTasks() { String id = stageContext.getId(); @@ -202,7 +238,7 @@ private void queryTasks() { TaskRegistry tasks = stageContext.getTaskRegistry(); List tasklets = Tasklets.newQueriers(tasks); executeTasklets(tasklets); - LOGGER.info("successfully queied all tasks in stage {}", id); + LOGGER.info("successfully queried all tasks in stage {}", id); } private void closeTasks() { diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/WorkloadProcessor.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/WorkloadProcessor.java index 37063314..360ca0b4 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/service/WorkloadProcessor.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/service/WorkloadProcessor.java @@ -118,7 +118,7 @@ public void process() { /* for strong consistency: a lock should be employed here */ if (!workloadContext.getState().equals(QUEUING)) throw new IllegalStateException( - "workload should be in the state of queuing"); + "workload should be in the state of queuing but " + workloadContext.getState().name()); String id = workloadContext.getId(); LOGGER.info("begin to process workload {}", id); try { diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractCommandTasklet.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractCommandTasklet.java index 50a81192..558447dc 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractCommandTasklet.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractCommandTasklet.java @@ -32,6 +32,8 @@ abstract class AbstractCommandTasklet extends AbstractHttpTasklet { private Class clazz; + protected long timeDrift = 0; /* real time drift between controller and driver */ + private static int tolerableTimeDrift = 300; /* tolerable time drift between controller and driver */ protected abstract void handleResponse(T response); @@ -49,7 +51,19 @@ protected void initObjectMapper() { } protected void issueCommand(String command) { - issueCommand(command, null); + int count = 3; + long timeStamp = System.currentTimeMillis(); + while (--count >= 0) { + issueCommand(command, String.valueOf(timeStamp)); + if (Math.abs(timeDrift) < tolerableTimeDrift) + break; + timeStamp = System.currentTimeMillis() + timeDrift / 2; + } + LOGGER.info("time drift between controller and driver-{} is {} mSec", + getDriver().getName(), timeDrift); + if (count < 0) + LOGGER.warn("time drift is still longer than tolerable time drift {} mSec after 3 times of synchronization", + tolerableTimeDrift); } protected void issueCommand(String command, String content) { diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractHttpTasklet.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractHttpTasklet.java index f6a2cc7e..9bd57d3b 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractHttpTasklet.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/AbstractHttpTasklet.java @@ -45,10 +45,10 @@ protected void initHttpClient() { context.setHttpClient(client); } - protected void closeHttpClient() { + protected synchronized void closeHttpClient() { HttpClient client = context.getHttpClient(); HttpClientUtil.disposeHttpClient(client); - context.setHttpClient(null); +// context.setHttpClient(null); } protected String issueHttpRequest(String command, String content) { diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Bootor.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Bootor.java index ac1952ac..cfdf7b4d 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Bootor.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Bootor.java @@ -48,11 +48,18 @@ public void execute() { @Override protected void handleResponse(PingResponse response) { - DriverInfo driver = getDriver(); - if (StringUtils.equals(response.getName(), driver.getName())) - return; - String msg = "expetect driver name {} dose not match the real name {}"; - LOGGER.debug(msg, driver.getName(), response.getName()); + long driverTime = 0; + DriverInfo driver = getDriver(); + try { + driverTime = Long.parseLong(response.getTimeStamp()); + } catch (NumberFormatException e) { + LOGGER.debug("time stamp of driver {} can not be formated", driver.getName()); + } + timeDrift = System.currentTimeMillis() - driverTime; + if (!StringUtils.equals(response.getName(), driver.getName())){ + String msg = "expetect driver name {} dose not match the real name {}"; + LOGGER.debug(msg, driver.getName(), response.getName()); + } } } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Querier.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Querier.java index 0b032948..67d733f7 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Querier.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Querier.java @@ -44,7 +44,11 @@ protected void execute() { String id = context.getMissionId(); do { sleep(); - issueCommand("query", id); + try{ + issueCommand("query", id); + }catch(Exception tle) { + LOGGER.warn("some unexpected exception occurs when ping drivers, but it's ignorable.", tle); + } } while (!context.getState().equals(FINISHED)); } @@ -59,6 +63,11 @@ private void sleep() { @Override protected void handleResponse(QueryResponse response) { + if (response == null) { + LOGGER.warn("no response gets from driver"); + return; + } + if (!response.isRunning()) context.setState(FINISHED); // stop querying Date time = response.getTime(); diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Aggregator.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Aggregator.java index 6926bbad..a5dc62a4 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Aggregator.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Aggregator.java @@ -65,6 +65,7 @@ public Metrics aggregate() { metrics.setThroughput(getThroughput()); metrics.setBandwidth(getBandwidth()); metrics.setAvgResTime(getAvgResTime()); + metrics.setAvgXferTime(getAvgXferTime()); metrics.setLatency(getLatency()); metrics.setRatio(metrics.getTotalSampleCount() > 0 ? (double) metrics .getSampleCount() / metrics.getTotalSampleCount() : 0D); @@ -94,6 +95,15 @@ private double getAvgResTime() { return sum / sampleCount; } + private double getAvgXferTime() { + if (sampleCount == 0) + return 0D; + double sum = 0D; + for (Metrics metrics : children) + sum += metrics.getAvgXferTime() * metrics.getSampleCount(); + return sum / sampleCount; + } + private Histogram getLatency() { if (!containsLatency) return null; diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Mark.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Mark.java index 46ad89bd..760523e4 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Mark.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Mark.java @@ -41,6 +41,7 @@ public class Mark implements Cloneable, Item { private int totalSampleCount; /* number of total samples */ private long rtSum; /* total response time */ + private long xtSum; /* total transfer time */ private long byteCount; /* total bytes transferred */ public Mark() { @@ -128,6 +129,14 @@ public void setRtSum(long rtSum) { this.rtSum = rtSum; } + public long getXtSum() { + return xtSum; + } + + public void setXtSum(long xtSum) { + this.xtSum = xtSum; + } + public long getByteCount() { return byteCount; } @@ -142,6 +151,7 @@ public void clear() { totalOpCount = 0; totalSampleCount = 0; rtSum = 0; + xtSum = 0; byteCount = 0; } @@ -150,6 +160,7 @@ public void addSample(Sample sample) { { sampleCount += 1; rtSum += sample.getTime(); + xtSum += sample.getXferTime(); byteCount += sample.getBytes(); } diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java index e65ecf75..eaa32e9a 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java @@ -47,6 +47,7 @@ public class Metrics implements Item, Cloneable { /* Metrics */ private double avgResTime; /* average response time */ + private double avgXferTime; /* average transfer time */ private double throughput; /* operation throughput */ private double bandwidth; /* network bandwidth */ @@ -141,6 +142,14 @@ public void setAvgResTime(double avgResTime) { this.avgResTime = avgResTime; } + public double getAvgXferTime() { + return avgXferTime; + } + + public void setAvgXferTime(double avgXferTime) { + this.avgXferTime = avgXferTime; + } + public double getThroughput() { return throughput; } @@ -202,6 +211,7 @@ public static Metrics convert(Mark mark, long window) { int sps = mark.getSampleCount(); int tsps = mark.getTotalSampleCount(); long rtSum = mark.getRtSum(); + long xtSum = mark.getXtSum(); long bytes = mark.getByteCount(); String type = getMetricsType(mark.getOpId(), mark.getOpType(), mark.getSampleType(), mark.getOpName()); @@ -213,6 +223,7 @@ public static Metrics convert(Mark mark, long window) { metrics.setByteCount(bytes); metrics.setWorkerCount(1); metrics.setAvgResTime(rtSum > 0 ? ((double) rtSum) / sps : 0); + metrics.setAvgXferTime(xtSum > 0 ? ((double) xtSum) / sps : 0); metrics.setThroughput(sps > 0 ? ((double) sps) / window * 1000 : 0); metrics.setBandwidth(bytes > 0 ? ((double) bytes) / window * 1000 : 0); return metrics; diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Sample.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Sample.java index 8d307e12..63286218 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Sample.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Sample.java @@ -36,19 +36,21 @@ public class Sample { private String sampleType; private long time; /* response time */ + private long xferTime; /* transfer time */ private long bytes; /* bytes transferred */ public Sample(Date timestamp, String opId, String opType, String sampleType, String opName, boolean succ) { - this(timestamp, opId, opType, sampleType, opName, succ, 0L, 0L); + this(timestamp, opId, opType, sampleType, opName, succ, 0L, 0L, 0L); } public Sample(Date timestamp, String opId, String opType, - String sampleType, String opName, boolean succ, long time, + String sampleType, String opName, boolean succ, long time, long xferTime, long bytes) { this.timestamp = timestamp; this.succ = succ; this.time = time; + this.xferTime = xferTime; this.bytes = bytes; this.opType = opType; this.sampleType = sampleType; @@ -104,6 +106,14 @@ public void setTime(long time) { this.time = time; } + public long getXferTime() { + return xferTime; + } + + public void setXferTime(long xferTime) { + this.xferTime = xferTime; + } + public long getBytes() { return bytes; } diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVLatencyExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVLatencyExporter.java index 13f0eba8..b09075e3 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVLatencyExporter.java +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVLatencyExporter.java @@ -22,6 +22,8 @@ import java.io.*; import com.intel.cosbench.bench.*; +import com.intel.cosbench.config.Work; +import com.intel.cosbench.model.StageInfo; /** * This class is to export response time histogram data into CSV format. @@ -35,20 +37,32 @@ class CSVLatencyExporter extends AbstractLatencyExporter { protected void writeHeader(Writer writer) throws IOException { StringBuilder buffer = new StringBuilder(); buffer.append("ResTime").append(','); - for (Metrics metrics : workload.getReport()) - writeOpType(buffer, metrics); + for (StageInfo stage : workload.getStageInfos()) { + writeOpType(buffer, stage); + } + buffer.setCharAt(buffer.length() - 1, '\n'); writer.write(buffer.toString()); } - private static void writeOpType(StringBuilder buffer, Metrics metrics) { - String opt = metrics.getOpName(); - String spt = metrics.getSampleType(); - if (spt.equals(opt)) - buffer.append(opt); - else - buffer.append(opt + '-' + spt); - buffer.append(',').append("(%)").append(','); + private static void writeOpType(StringBuilder buffer, StageInfo stage) { + for (Metrics metrics : stage.getReport()) { + String opt = metrics.getOpName(); + String spt = metrics.getSampleType(); + if (spt.equals(opt)){ /*just append normal stage*/ + String workName = null; + int workIdx = 1; + for (Work work : stage.getStage().getWorks()){ + if (work.getOperationIDs().contains(metrics.getOpId())) { + workName = "w" + workIdx + "-" + work.getName(); + break; + } + workIdx++; + } + buffer.append(stage.getId() + "-" + workName + "-" + opt); + buffer.append(',').append("(%)").append(','); + } + } } @Override @@ -61,14 +75,20 @@ protected void writeHistogram(Writer writer, int idx) throws IOException { else buffer.append("+INF"); buffer.append(','); - int metricsIdx = 0; - for (Metrics metrics : workload.getReport()) { - int count = metrics.getLatency().getHistoData()[idx]; - buffer.append(count).append(','); - accs[metricsIdx] += count; - double per = accs[metricsIdx] / ((double) sums[metricsIdx]); - buffer.append(RATIO.format(per)).append(','); - metricsIdx++; + int metricsIdx = -1; + for (StageInfo stage : workload.getStageInfos()) { + for (Metrics metrics : stage.getReport()) { + metricsIdx++; + if (!metrics.getOpName().equals(metrics.getSampleType())) { + continue; /*skip for special work*/ + } + int count = metrics.getLatency().getHistoData()[idx]; + buffer.append(count).append(','); + accs[metricsIdx] += count; + double per = sums[metricsIdx] != 0 ? + accs[metricsIdx] / ((double) sums[metricsIdx]) : 0; + buffer.append(RATIO.format(per)).append(','); + } } buffer.setCharAt(buffer.length() - 1, '\n'); writer.write(buffer.toString()); diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVMatrixExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVMatrixExporter.java index 247e9fe3..ac5a1a06 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVMatrixExporter.java +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVMatrixExporter.java @@ -42,6 +42,7 @@ protected void writeHeader(Writer writer) throws IOException { buffer.append("Byte-Count").append(','); buffer.append("Worker-Count").append(','); buffer.append("Avg-ResTime").append(','); + buffer.append("Avg-ProcTime").append(','); buffer.append("60%-ResTime").append(','); buffer.append("80%-ResTime").append(','); buffer.append("90%-ResTime").append(','); @@ -77,6 +78,14 @@ protected void writeMetrics(Writer writer, StageInfo stage, else buffer.append("N/A"); buffer.append(','); + + double pt = r - metrics.getAvgXferTime(); + if (pt > 0) + buffer.append(NUM.format(pt)); + else + buffer.append("N/A"); + buffer.append(','); + writeLatencyInfo(buffer, metrics.getLatency()); buffer.append(NUM.format(metrics.getThroughput())).append(','); buffer.append(NUM.format(metrics.getBandwidth())).append(','); diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVStageExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVStageExporter.java index 05a5d5a3..60a89f25 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVStageExporter.java +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVStageExporter.java @@ -47,13 +47,14 @@ protected void writeHeader(Writer writer) throws IOException { buffer.append("Op-Count").append(suffix); buffer.append("Byte-Count").append(suffix); buffer.append("Avg-ResTime").append(suffix); + buffer.append("Avg-ProcTime").append(suffix); buffer.append("Throughput").append(suffix); buffer.append("Bandwidth").append(suffix); buffer.append("Succ-Ratio").append(suffix); buffer.append("Version-Info"); buffer.append(',').append(',').append('\n').append(','); - for (int i = 0; i < 6; i++) - // 6 metrics + for (int i = 0; i < 7; i++) + // 7 metrics for (Metrics metrics : snapshots[0].getReport()) buffer.append( StringUtils.join(new Object[] { @@ -93,6 +94,15 @@ protected void writeMetrics(Writer writer, Snapshot snapshot) buffer.append("N/A"); buffer.append(','); } + /* Transfer Time */ + for (Metrics metrics : report) { + double pt = metrics.getAvgResTime() - metrics.getAvgXferTime(); + if (pt > 0) + buffer.append(NUM.format(pt)); + else + buffer.append("N/A"); + buffer.append(','); + } /* Throughput */ for (Metrics metrics : report) buffer.append(NUM.format(metrics.getThroughput())).append(','); diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVWorkloadExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVWorkloadExporter.java index 71ead26e..848c3ea8 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVWorkloadExporter.java +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVWorkloadExporter.java @@ -42,6 +42,7 @@ protected void writeHeader(Writer writer) throws IOException { buffer.append("Op-Count").append(','); buffer.append("Byte-Count").append(','); buffer.append("Avg-ResTime").append(','); + buffer.append("Avg-ProcTime").append(','); buffer.append("60%-ResTime").append(','); buffer.append("80%-ResTime").append(','); buffer.append("90%-ResTime").append(','); @@ -77,6 +78,14 @@ protected void writeMetrics(Writer writer, Metrics metrics, StageInfo stage) else buffer.append("N/A"); buffer.append(','); + + double pt = r - metrics.getAvgXferTime(); + if (pt > 0) + buffer.append(NUM.format(pt)); + else + buffer.append("N/A"); + buffer.append(','); + writeLatencyInfo(buffer, metrics.getLatency()); buffer.append(NUM.format(metrics.getThroughput())).append(','); buffer.append(NUM.format(metrics.getBandwidth())).append(','); @@ -114,6 +123,7 @@ protected void writeMetrics(Writer writer, StageInfo stage) buffer.append("N/A").append(','); buffer.append("N/A").append(','); buffer.append("N/A").append(','); + buffer.append("N/A").append(','); buffer.append(stage.getState().name().toLowerCase()).append(','); for (StateInfo state : stage.getStateHistory()) { buffer.append( diff --git a/dev/cosbench-core/src/com/intel/cosbench/model/DriverInfo.java b/dev/cosbench-core/src/com/intel/cosbench/model/DriverInfo.java index 2db02600..565c7ea3 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/model/DriverInfo.java +++ b/dev/cosbench-core/src/com/intel/cosbench/model/DriverInfo.java @@ -23,4 +23,8 @@ public interface DriverInfo { public String getUrl(); + public void setAliveState(boolean aliveState); + + public boolean getAliveState(); + } diff --git a/dev/cosbench-core/src/com/intel/cosbench/protocol/PingResponse.java b/dev/cosbench-core/src/com/intel/cosbench/protocol/PingResponse.java index 9dfd32f9..608b1019 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/protocol/PingResponse.java +++ b/dev/cosbench-core/src/com/intel/cosbench/protocol/PingResponse.java @@ -27,8 +27,9 @@ public class PingResponse extends Response { private String name; /* driver name */ private String address; /* driver address */ + private String timeStamp; /* driver current time */ - public PingResponse() { + public PingResponse() { /* empty */ } @@ -47,5 +48,13 @@ public String getAddress() { public void setAddress(String address) { this.address = address; } + + public String getTimeStamp() { + return timeStamp; + } + + public void setTimeStamp(String timeStamp) { + this.timeStamp = timeStamp; + } } diff --git a/dev/cosbench-driver-web/WEB-INF/freemarker/header.ftl b/dev/cosbench-driver-web/WEB-INF/freemarker/header.ftl index 67e19980..71fa8012 100644 --- a/dev/cosbench-driver-web/WEB-INF/freemarker/header.ftl +++ b/dev/cosbench-driver-web/WEB-INF/freemarker/header.ftl @@ -1,6 +1,6 @@ <#-- end of header --> diff --git a/dev/cosbench-driver-web/WEB-INF/freemarker/metrics.ftl b/dev/cosbench-driver-web/WEB-INF/freemarker/metrics.ftl index 2f24ff02..3c66caff 100644 --- a/dev/cosbench-driver-web/WEB-INF/freemarker/metrics.ftl +++ b/dev/cosbench-driver-web/WEB-INF/freemarker/metrics.ftl @@ -5,6 +5,7 @@ + @@ -67,6 +68,14 @@ ${mInfo.avgResTime?string("0.##")} ms +
Authentication - + + - +
Storage - + + + + + + - +
Driver Name URLIsAlive Link
${dInfo_index + 1} ${dInfo.name} ${dInfo.url}
 view details
95%-ResTime99%-ResTime100%-ResTimeAvg-ProcTimeThroughputBandwidthSucc-Ratio + <#assign procTime = mInfo.avgResTime - mInfo.avgXferTime> + <#if procTime == 0> + N/A + <#else> + ${procTime?string("0.##")} ms + + ${mInfo.throughput?string("0.##")} op/s Op-Count Byte-Count Avg-ResTimeAvg-ProcTime Throughput Bandwidth Succ-Ratio + <#assign procTime = mInfo.avgResTime - mInfo.avgXferTime> + <#if procTime == 0> + N/A + <#else> + ${procTime?string("0.##")} ms + + ${mInfo.throughput?string("0.##")} op/s <#assign bw = mInfo.bandwidth > diff --git a/dev/cosbench-controller-web/WEB-INF/freemarker/timeline-metrics.ftl b/dev/cosbench-controller-web/WEB-INF/freemarker/timeline-metrics.ftl index a0931948..1761e56b 100644 --- a/dev/cosbench-controller-web/WEB-INF/freemarker/timeline-metrics.ftl +++ b/dev/cosbench-controller-web/WEB-INF/freemarker/timeline-metrics.ftl @@ -6,6 +6,7 @@ Op-Count Byte-Count Avg-ResTimeAvg-ProcTime Throughput Bandwidth Succ-Ratio + <#assign procTime = mInfo.avgResTime - mInfo.avgXferTime> + <#if procTime == 0 > + N/A + <#else> + ${procTime?string("0.##")} ms${mInfo.throughput?string("0.##")} op/s Op-Count Byte-Count Avg-ResTimeAvg-ProcTime Throughput Bandwidth Succ-Ratio + <#assign procTime = mInfo.avgResTime - mInfo.avgXferTime> + <#if procTime == 0> + N/A + <#else> + ${procTime?string("0.##")} ms + + ${mInfo.throughput?string("0.##")} op/s <#assign bw = mInfo.bandwidth > diff --git a/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/AbstractCommandHandler.java b/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/AbstractCommandHandler.java index 4924cef6..ada6552c 100644 --- a/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/AbstractCommandHandler.java +++ b/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/AbstractCommandHandler.java @@ -75,7 +75,7 @@ public ModelAndView handleRequest(HttpServletRequest req, try { response = process(req, res); } catch (BadRequestException bre) { - response = new Response(400, "unrecognized request"); + response = new Response(400, "unrecognized request: " + req.toString()); } catch (NotFoundException nfe) { response = new Response(404, "mission not found"); } catch (IllegalStateException ise) { diff --git a/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/PingHandler.java b/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/PingHandler.java index 9dd3285f..306c7a37 100644 --- a/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/PingHandler.java +++ b/dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/PingHandler.java @@ -17,11 +17,18 @@ package com.intel.cosbench.driver.handler; +import java.io.IOException; +import java.sql.Date; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Scanner; + import javax.servlet.http.*; import com.intel.cosbench.model.DriverInfo; import com.intel.cosbench.protocol.*; import com.intel.cosbench.service.DriverService; +import com.intel.cosbench.web.BadRequestException; public class PingHandler extends AbstractCommandHandler { @@ -32,12 +39,34 @@ public void setDriver(DriverService driver) { } @Override - protected Response process(HttpServletRequest req, HttpServletResponse res) { - PingResponse response = new PingResponse(); + protected Response process(HttpServletRequest req, HttpServletResponse res) + throws Exception { + Scanner scanner = new Scanner(req.getInputStream()); + setSysTime(getControllerTime(scanner)); + + PingResponse response = new PingResponse(); DriverInfo info = driver.getDriverInfo(); response.setName(info.getName()); response.setAddress(info.getUrl()); + response.setTimeStamp(String.valueOf(System.currentTimeMillis())); return response; } + + private long getControllerTime(Scanner scanner) throws NumberFormatException { + if (!scanner.hasNext()) + throw new BadRequestException(); + return Long.parseLong(scanner.next()); + } + + private void setSysTime(long ctrTime) throws IOException { + DateFormat dateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String[] cmd = {"date", "-s", dateTime.format(new Date(ctrTime))}; + String osType = System.getProperty("os.name").toLowerCase(); + if (osType.contains("linux")) { + Runtime.getRuntime().exec(cmd); + } else { + /* skip for non linux system */ + } + } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WatchDog.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WatchDog.java index 154a4b66..085ef271 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WatchDog.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WatchDog.java @@ -64,7 +64,9 @@ public void run() { secs -= 5; } if (!cancel) { - workerContext.getStorageApi().abort(); +// workerContext.getStorageApi().abort(); + workerContext.disposeRuntime(); + LOGGER.debug("work agent {} has been alerted for timeout", idx); } else LOGGER.debug("work agent {} has completed before timeout", idx); diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java index 255dd3bf..ef7bbe0f 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java @@ -49,12 +49,13 @@ class WorkAgent extends AbstractAgent implements Session, OperationListener { private long interval; /* interval between check points */ private int totalOps; /* total operations to be performed */ +// private int op_count; private long totalBytes; /* total bytes to be transferred */ private OperationPicker operationPicker; private OperatorRegistry operatorRegistry; - private boolean isFinished = false; +// private boolean isFinished = false; private WatchDog dog = new WatchDog(); private Status currMarks = new Status(); /* for snapshots */ @@ -156,25 +157,28 @@ private void initMarks() { private void doWork() { doSnapshot(); - while (!isFinished) + while (!workerContext.isFinished()) try { performOperation(); } catch (AbortedException ae) { if (lrsample > frsample) doSummary(); - isFinished = true; + workerContext.setFinished(true); } doSnapshot(); } private void performOperation() { + if(workerContext.getAuthApi() == null || workerContext.getStorageApi() == null) + throw new AbortedException(); + lbegin = System.currentTimeMillis(); Random random = workerContext.getRandom(); String op = operationPicker.pickOperation(random); OperatorContext context = operatorRegistry.getOperator(op); context.getOperator().operate(this); } - + @Override public void onSampleCreated(Sample sample) { curr = sample.getTimestamp().getTime(); @@ -214,11 +218,13 @@ private void doSnapshot() { @Override public void onOperationCompleted(Result result) { curr = result.getTimestamp().getTime(); +/* */ String type = getMarkType(result.getOpId(), result.getOpType(), result.getSampleType(), result.getOpName()); currMarks.getMark(type).addOperation(result); if (lop >= begin && lop < end && curr > begin && curr <= end) globalMarks.getMark(type).addOperation(result); +/* */ lop = curr; // last operation performed trySummary(); // make a summary report if necessary } @@ -229,18 +235,23 @@ private void trySummary() { && (totalBytes <= 0 || getTotalBytes() < totalBytes)) // bytes return; // not finished doSummary(); - isFinished = true; + + workerContext.setFinished(true); } private void doSummary() { +/* */ long window = lrsample - frsample; Report report = new Report(); for (Mark mark : globalMarks) report.addMetrics(Metrics.convert(mark, window)); workerContext.setReport(report); +/* */ } private int getTotalOps() { +// return ++op_count; + int sum = 0; for (Mark mark : globalMarks) sum += mark.getTotalOpCount(); diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/HistogramIntGenerator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/HistogramIntGenerator.java index e835546b..80c2c106 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/HistogramIntGenerator.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/HistogramIntGenerator.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.math.RandomUtils; import com.intel.cosbench.config.ConfigException; +//import com.intel.cosbench.driver.generator.RangeIntGenerator.TestThread; /** * @@ -101,13 +102,22 @@ private static HistogramIntGenerator tryParse(String pattern) { String[] args = StringUtils.split(pattern, ','); ArrayList bucketsList = new ArrayList(); for (String arg : args) { + int v1 = StringUtils.indexOf(arg, '|'); + int v2 = StringUtils.lastIndexOf(arg, '|'); + boolean isOpenRange = ((v2 - v1) == 1) ? true : false; String[] values = StringUtils.split(arg, '|'); - if (values.length != 3) { + int lower,upper,weight; + if (isOpenRange) { + lower = Integer.parseInt(values[0]); + upper = UniformIntGenerator.getMAXupper(); + weight = Integer.parseInt(values[1]); + } else if (values.length != 3) { throw new IllegalArgumentException(); + } else { + lower = Integer.parseInt(values[0]); + upper = Integer.parseInt(values[1]); + weight = Integer.parseInt(values[2]); } - int lower = Integer.parseInt(values[0]); - int upper = Integer.parseInt(values[1]); - int weight = Integer.parseInt(values[2]); bucketsList.add(new Bucket(lower, upper, weight)); } if (bucketsList.isEmpty()) { diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/RangeIntGenerator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/RangeIntGenerator.java index 919edb7b..1ff7fbba 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/RangeIntGenerator.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/RangeIntGenerator.java @@ -31,15 +31,15 @@ public class RangeIntGenerator implements IntGenerator { private int lower; private int upper; - private AtomicInteger cursor; + private AtomicInteger cursors[] = null; static class TestThread extends Thread { private int all; private int idx; - private RangeIntGenerator generator; + private IntGenerator generator; private Random rnd; - public TestThread(RangeIntGenerator gen, int idx, int all) { + public TestThread(IntGenerator gen, int idx, int all) { this.all = all; this.idx = idx; this.setName("Thread[" + idx + "]"); @@ -60,7 +60,7 @@ public void run() { System.out.println(this.getName() + ": " + answer + " I did get this before!"); } results.add(answer); - sleep(1); + sleep(5); } } catch (InterruptedException ie) { ie.printStackTrace(); @@ -69,7 +69,7 @@ public void run() { } public static void main(String[] args) { - final String pattern = "r(50,100)"; + final String pattern = "r(51,1000)"; final int all = 5; int i = 0; Vector threads = new Vector(); @@ -95,10 +95,20 @@ public RangeIntGenerator(int lower, int upper) { throw new IllegalArgumentException(); this.lower = lower; this.upper = upper; - - this.cursor = new AtomicInteger(0); + } + private synchronized void init(int all) { + if(cursors != null) + return; + + this.cursors = new AtomicInteger[all]; + + for (int i = 0; i= idx - 1 ? idx - 1 : extra); int segment = base + (extra >= idx ? 1 : 0); - - cursor.set(cursor.get() % (segment)); - - return lower + offset + cursor.getAndIncrement(); + + return lower + offset + cursors[idx-1].getAndIncrement() % segment; } public static RangeIntGenerator parse(String pattern) { diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/UniformIntGenerator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/UniformIntGenerator.java index 3488e1c6..32fe32d4 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/UniformIntGenerator.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/UniformIntGenerator.java @@ -34,6 +34,7 @@ class UniformIntGenerator implements IntGenerator { private int lower; private int upper; + private static int MAXupper = Integer.MAX_VALUE; public UniformIntGenerator(int lower, int upper) { if (lower <= 0 || upper <= 0 || lower > upper) @@ -74,8 +75,12 @@ private static UniformIntGenerator tryParse(String pattern) { pattern = StringUtils.substringBetween(pattern, "(", ")"); String[] args = StringUtils.split(pattern, ','); int lower = Integer.parseInt(args[0]); - int upper = Integer.parseInt(args[1]); + int upper = (args.length == 2) ? Integer.parseInt(args[1]) : MAXupper; return new UniformIntGenerator(lower, upper); } + public static int getMAXupper () { + return MAXupper; + } + } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java new file mode 100644 index 00000000..78bb419a --- /dev/null +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java @@ -0,0 +1,56 @@ +package com.intel.cosbench.driver.generator; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.io.input.CountingInputStream; + +/** + * This class is to record the time of data transfer + * + * + */ + +public class XferCountingInputStream extends CountingInputStream{ + private long xferStart = 0L; + private long xferEnd = 0L; + private boolean isFirstByte = true; + + public XferCountingInputStream(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + int result = super.read(); + recordTime(); + return result; + } + + @Override + public int read(byte[] b) throws IOException { + int result = super.read(b); + recordTime(); + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = super.read(b, off, len); + recordTime(); + return result; + } + + private void recordTime() { + if (this.isFirstByte) { + this.xferStart = System.currentTimeMillis(); + this.isFirstByte = false; + } + this.xferEnd = System.currentTimeMillis(); + } + + public long getXferTime() { + long xferTime = this.xferEnd - this.xferStart; + return xferTime > 0 ? xferTime : 0L; + } + +} diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/DriverContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/DriverContext.java index 5f182472..d22849d2 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/DriverContext.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/DriverContext.java @@ -29,6 +29,7 @@ public class DriverContext implements DriverInfo { private String name; private String url; + private boolean aliveState; public DriverContext() { /* empty */ @@ -50,4 +51,12 @@ public void setUrl(String url) { this.url = url; } + public void setAliveState(boolean aliveState) { + this.aliveState = aliveState; + } + + public boolean getAliveState(){ + return aliveState; + } + } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java index 1542d5da..0c215c93 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java @@ -204,14 +204,16 @@ public void addListener(MissionListener listener) { @Override public void disposeRuntime() { - for (WorkerContext worker : workerRegistry) - worker.disposeRuntime(); - config = null; - future = null; - operationPicker = null; - operatorRegistry = null; - listeners = null; - logManager.dispose(); + if(MissionState.isStopped(state)) { + for (WorkerContext worker : workerRegistry) + worker.disposeRuntime(); + config = null; + future = null; + operationPicker = null; + operatorRegistry = null; + listeners = null; + logManager.dispose(); + } } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/WorkerContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/WorkerContext.java index 6b11ac1c..4b200346 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/WorkerContext.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/WorkerContext.java @@ -44,6 +44,8 @@ public class WorkerContext implements WorkerInfo { private volatile boolean error = false; private volatile boolean aborted = false; + private volatile boolean finished = false; + /* Each worker starts with an empty snapshot */ private transient volatile Snapshot snapshot = new Snapshot(); /* Each worker starts with an empty report */ @@ -53,6 +55,7 @@ public class WorkerContext implements WorkerInfo { /* Each worker has its private required version */ private volatile int version = 0; private volatile int runlen = 0; + public WorkerContext() { /* empty */ @@ -174,13 +177,26 @@ public Random getRandom() { return random; } + public boolean isFinished() { + return finished; + } + + public void setFinished(boolean finished) { + this.finished = finished; + } + @Override - public void disposeRuntime() { - authApi.dispose(); - authApi = null; - storageApi.dispose(); - storageApi = null; - random = null; + public synchronized void disposeRuntime() { + if(authApi != null) { + authApi.dispose(); + authApi = null; + } + if(storageApi != null) { + storageApi.dispose(); + storageApi = null; + } + finished = true; +// random = null; // snapshot = new Snapshot(); // logger = null; } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java index 3bdb7c92..324ef104 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java @@ -70,6 +70,20 @@ public void operate(Session session) { int all = session.getTotalWorkers(); operate(idx, all, session); } + + protected static void doLogInfo(Logger logger, String message) { + if (logger != null) + logger.info(message); + else + AbstractOperator.LOGGER.info(message); + } + + protected static void doLogDebug(Logger logger, String message) { + if (logger != null) + logger.debug(message); + else + AbstractOperator.LOGGER.debug(message); + } protected static void doLogWarn(Logger logger, String message) { if (logger != null) diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java index 922667df..5d7c7da1 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java @@ -88,7 +88,7 @@ public static Sample doDelete(String conName, String objName, Date now = new Date(end); return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(), - op.getName(), true, end - start, 0L); + op.getName(), true, end - start, 0L, 0L); } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java index b8a1328f..6c252311 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java @@ -25,12 +25,12 @@ import java.util.Random; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.CountingInputStream; import com.intel.cosbench.api.storage.StorageInterruptedException; import com.intel.cosbench.bench.Result; import com.intel.cosbench.bench.Sample; import com.intel.cosbench.config.Config; +import com.intel.cosbench.driver.generator.XferCountingInputStream; import com.intel.cosbench.driver.util.ContainerPicker; import com.intel.cosbench.driver.util.FilePicker; import com.intel.cosbench.driver.util.HashUtil; @@ -135,7 +135,7 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam if (Thread.interrupted()) throw new AbortedException(); - CountingInputStream cin = new CountingInputStream(in); + XferCountingInputStream cin = new XferCountingInputStream(in); long start = System.currentTimeMillis(); @@ -155,6 +155,6 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam Date now = new Date(end); return new Sample(now, getId(), getOpType(), getSampleType(), - getName(), true, end - start, cin.getByteCount()); + getName(), true, end - start, cin.getXferTime(), cin.getByteCount()); } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java index 1a3b7f0c..9f081310 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java @@ -43,6 +43,8 @@ class Reader extends AbstractOperator { private boolean hashCheck = false; private ObjectPicker objPicker = new ObjectPicker(); + + private byte buffer[] = new byte[1024*1024]; public Reader() { /* empty */ @@ -81,12 +83,17 @@ private Sample doRead(OutputStream out, String conName, String objName, CountingOutputStream cout = new CountingOutputStream(out); long start = System.currentTimeMillis(); - + long xferTime = 0L; + long xferTimeCheck = 0L; try { + doLogDebug(session.getLogger(), "Read Object " + conName + "/" + objName); in = session.getApi().getObject(conName, objName, config); - if (!hashCheck) - IOUtils.copyLarge(in, cout); - else if (!validateChecksum(conName, objName, session, in, cout)) + long xferStart = System.currentTimeMillis(); + if (!hashCheck){ + copyLarge(in, cout); + long xferEnd = System.currentTimeMillis(); + xferTime = xferEnd - xferStart; + } else if (!validateChecksum(conName, objName, session, in, cout, xferTimeCheck)) return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false); } catch (StorageInterruptedException sie) { @@ -98,16 +105,26 @@ else if (!validateChecksum(conName, objName, session, in, cout)) IOUtils.closeQuietly(in); IOUtils.closeQuietly(cout); } - long end = System.currentTimeMillis(); Date now = new Date(end); return new Sample(now, getId(), getOpType(), getSampleType(), - getName(), true, end - start, cout.getByteCount()); + getName(), true, end - start, hashCheck ? xferTimeCheck : xferTime, cout.getByteCount()); } + public OutputStream copyLarge(InputStream input, OutputStream output) + throws IOException + { + for(int n = 0; -1 != (n = input.read(buffer));) + { + output.write(buffer, 0, n); + } + + return output; + } + private static boolean validateChecksum(String conName, String objName, - Session session, InputStream in, OutputStream out) + Session session, InputStream in, OutputStream out, long xferTimeCheck) throws IOException { HashUtil util; try { @@ -119,6 +136,8 @@ private static boolean validateChecksum(String conName, String objName, String storedHash = new String(); String calculatedHash = new String(); + + long xferStart = System.currentTimeMillis(); int br1 = in.read(buf1); if (br1 <= hashLen) { @@ -155,7 +174,8 @@ private static boolean validateChecksum(String conName, String objName, br1 = br2; } } - + xferTimeCheck = System.currentTimeMillis() - xferStart; + if (!calculatedHash.equals(storedHash)) { if (storedHash.startsWith(HashUtil.GUARD)) { String err = diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java index e6e3d7b9..114af4cc 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java @@ -21,11 +21,11 @@ import java.util.*; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.CountingInputStream; import com.intel.cosbench.api.storage.StorageInterruptedException; import com.intel.cosbench.bench.*; import com.intel.cosbench.config.Config; +import com.intel.cosbench.driver.generator.XferCountingInputStream; import com.intel.cosbench.driver.generator.RandomInputStream; import com.intel.cosbench.driver.util.*; import com.intel.cosbench.service.AbortedException; @@ -86,12 +86,12 @@ public static Sample doWrite(InputStream in, long length, String conName, String objName, Config config, Session session, Operator op) { if (Thread.interrupted()) throw new AbortedException(); - - CountingInputStream cin = new CountingInputStream(in); - + + XferCountingInputStream cin = new XferCountingInputStream(in); long start = System.currentTimeMillis(); try { + doLogDebug(session.getLogger(), "Write Object " + conName + "/" + objName); session.getApi() .createObject(conName, objName, cin, length, config); } catch (StorageInterruptedException sie) { @@ -105,10 +105,9 @@ public static Sample doWrite(InputStream in, long length, String conName, } long end = System.currentTimeMillis(); - Date now = new Date(end); return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(), - op.getName(), true, end - start, cin.getByteCount()); + op.getName(), true, end - start, cin.getXferTime(), cin.getByteCount()); } /* * public static Sample doWrite(byte[] data, String conName, String objName, diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/service/MissionHandler.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/service/MissionHandler.java index f2428b24..f78b4097 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/service/MissionHandler.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/service/MissionHandler.java @@ -218,7 +218,7 @@ public void login() { /* for strong consistency: a lock should be employed here */ if (!missionContext.getState().equals(SUBMITTED)) throw new IllegalStateException( - "mission should be in the state of submitted"); + "mission should be in the state of submitted but " + missionContext.getState().name()); String id = missionContext.getId(); LOGGER.debug("begin to auth mission {}", id); try { @@ -255,6 +255,7 @@ private void performLogin() { missionContext.setState(AUTHED); } + @SuppressWarnings("unused") private void setAllWorkersAuthContext(AuthContext authContext) { for (WorkerContext workerContext : missionContext.getWorkerRegistry()) workerContext.getStorageApi().setAuthContext(authContext); @@ -264,13 +265,13 @@ private void setAllWorkersAuthContext(AuthContext authContext) { * Returns Size 1 list of Agents * @param workerContext to create Agent for */ + @SuppressWarnings("unused") private List createAuthAgentFromContext(WorkerContext workerContext) { List agents = new ArrayList(); agents.add(Agents.newAuthAgent(retry, workerContext)); return agents; } - @SuppressWarnings("unused") private List createAuthAgents() { List agents = new ArrayList(); for (WorkerContext workerContext : missionContext.getWorkerRegistry()) @@ -291,7 +292,7 @@ public void stress() { /* no need to shutdown agents again */ boolean shutdownNow = false; abortAgents(shutdownNow); - missionContext.setState(ABORTED); +// missionContext.setState(ABORTED); return; } catch (AbortedException ae) { /* have to shutdown agents now */ @@ -398,21 +399,27 @@ public void abort() { private void abortAgents(boolean shutdownNow) { Thread.interrupted(); // clear interruption status - if (shutdownNow) - executor.shutdownNow(); // abort agents - else - executor.shutdown(); - if (!awaitTermination(5) && !awaitTermination(10)) - awaitTermination(30); - String id = missionContext.getId(); - if (!executor.isTerminated()) - LOGGER.warn("fail to abort agents for mission {}", id); - else - LOGGER.info("all agents have been aborted in mission {}", id); - /* - * Consider the mission aborted even if its agents have not. - */ - LOGGER.info("mission {} appears to be aborted", id); // agents aborted + + executor.shutdown(); + try { + // Wait a few seconds for existing tasks to terminate + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + executor.shutdownNow(); + + String id = missionContext.getId(); + + if (!awaitTermination(5) && !awaitTermination(10) && !awaitTermination(30)) + LOGGER.warn("fail to abort agents for mission {}", id); + else + LOGGER.info("all agents have been aborted in mission {}", id); + + LOGGER.info("mission {} appears to be aborted", id); // agents aborted + } + + } catch (InterruptedException ie) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } } private boolean awaitTermination(int seconds) { diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/util/Defaults.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/util/Defaults.java index dfb4d257..1e6fcbbf 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/util/Defaults.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/util/Defaults.java @@ -19,11 +19,11 @@ interface Defaults { - String CONTAINER_PREFIX = "mycontainers_"; + String CONTAINER_PREFIX = "mycontainers"; String CONTAINER_SUFFIX = null; - String OBJECT_PREFIX = "myobjects_"; + String OBJECT_PREFIX = "myobjects"; String OBJECT_SUFFIX = null; diff --git a/dev/cosbench-http/src/com/intel/cosbench/client/http/HttpClientUtil.java b/dev/cosbench-http/src/com/intel/cosbench/client/http/HttpClientUtil.java index 9c0268b5..f099708c 100644 --- a/dev/cosbench-http/src/com/intel/cosbench/client/http/HttpClientUtil.java +++ b/dev/cosbench-http/src/com/intel/cosbench/client/http/HttpClientUtil.java @@ -131,8 +131,11 @@ private static ClientConnectionManager createClientConnManager() * the HTTP client to be disposed. */ public static void disposeHttpClient(HttpClient client) { - ClientConnectionManager manager = client.getConnectionManager(); - manager.shutdown(); + if(client != null) { + ClientConnectionManager manager = client.getConnectionManager(); + manager.shutdown(); + client = null; + } } public static HttpGet makeHttpGet(String url) { diff --git a/dev/cosbench-httpauth/src/com/intel/cosbench/api/httpauth/HttpAuthTest.java b/dev/cosbench-httpauth/src/com/intel/cosbench/api/httpauth/HttpAuthTest.java index f092595d..99518827 100644 --- a/dev/cosbench-httpauth/src/com/intel/cosbench/api/httpauth/HttpAuthTest.java +++ b/dev/cosbench-httpauth/src/com/intel/cosbench/api/httpauth/HttpAuthTest.java @@ -20,7 +20,6 @@ import static com.intel.cosbench.client.httpauth.HttpAuthConstants.*; import org.apache.http.Header; -import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.auth.*; diff --git a/dev/cosbench-swift/src/com/intel/cosbench/api/swift/SwiftStorage.java b/dev/cosbench-swift/src/com/intel/cosbench/api/swift/SwiftStorage.java index 714bcbee..b125bcdc 100644 --- a/dev/cosbench-swift/src/com/intel/cosbench/api/swift/SwiftStorage.java +++ b/dev/cosbench-swift/src/com/intel/cosbench/api/swift/SwiftStorage.java @@ -47,6 +47,7 @@ class SwiftStorage extends NoneStorage { private int timeout; // connection and socket timeout private String token; private String storage_url; + private String policy; public SwiftStorage() { /* empty */ @@ -59,10 +60,12 @@ public void init(Config config, Logger logger) { timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); token = config.get(AUTH_TOKEN_KEY, AUTH_TOKEN_DEFAULT); storage_url = config.get(STORAGE_URL_KEY, STORAGE_URL_DEFAULT); + policy = config.get(POLICY_KEY, POLICY_DEFAULT); parms.put(CONN_TIMEOUT_KEY, timeout); parms.put(AUTH_TOKEN_KEY, token); parms.put(STORAGE_URL_KEY, storage_url); + parms.put(POLICY_KEY, policy); logger.debug("using storage config: {}", parms); @@ -81,11 +84,17 @@ public void setAuthContext(AuthContext info) { } try { - client.init(token, storage_url); + client.init(token, storage_url, policy); } catch (Exception e) { throw new StorageException(e); } - logger.debug("using auth token: {}, storage url: {}", token, storage_url); + logger.debug(new StringBuffer() + .append("using auth token: ") + .append(token) + .append(", storage url: ") + .append(storage_url) + .append(", storage policy: ") + .append(policy).toString()); } @Override diff --git a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java index 3146a88f..1d1f82a1 100644 --- a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java +++ b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java @@ -17,7 +17,7 @@ package com.intel.cosbench.client.swift; -import static com.intel.cosbench.client.swift.SwiftConstants.X_AUTH_TOKEN; +import static com.intel.cosbench.client.swift.SwiftConstants.*; import static org.apache.http.HttpStatus.*; import java.io.*; @@ -33,11 +33,12 @@ public class SwiftClient { - private static boolean REPORT_DELETE_ERROR = false; + private static boolean REPORT_DELETE_ERROR = false; /* user context */ private String authToken; private String storageURL; + private String policy; /* HTTP client */ private HttpClient client; @@ -68,9 +69,10 @@ public void abort() { method = null; } - public void init(String authToken, String storageURL) { + public void init(String authToken, String storageURL, String policy) { this.authToken = authToken; this.storageURL = storageURL; + this.policy = policy; } public SwiftAccount getAccountInfo() throws IOException, SwiftException { @@ -125,6 +127,8 @@ public void createContainer(String container) throws IOException, method = HttpClientUtil.makeHttpPut(getContainerPath(container)); method.setHeader(X_AUTH_TOKEN, authToken); + if(policy != null) + method.setHeader(X_STORAGE_POLICY, policy); response = new SwiftResponse(client.execute(method)); if (response.getStatusCode() == SC_CREATED) { logger.info("SUCCESS"); diff --git a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftConstants.java b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftConstants.java index 5c5c5063..648d954f 100644 --- a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftConstants.java +++ b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftConstants.java @@ -37,6 +37,9 @@ public interface SwiftConstants { String STORAGE_URL_KEY = "storage_url"; String STORAGE_URL_DEFAULT = "http://127.0.0.1:8080/auth/v1.0"; + String POLICY_KEY = "policy"; + String POLICY_DEFAULT = null; + // -------------------------------------------------------------------------- // Swift RESTful API // -------------------------------------------------------------------------- @@ -44,6 +47,8 @@ public interface SwiftConstants { String X_STORAGE_URL = "X-Storage-Url"; String X_AUTH_TOKEN = "X-Auth-Token"; + + String X_STORAGE_POLICY = "X-Storage-Policy"; String X_CONTAINER_OBJECT_COUNT = "X-Container-Object-Count"; diff --git a/release/conf/cdmi-base-config-sample.xml b/release/conf/cdmi-base-config-sample.xml index 80c6a5e1..57af26a2 100644 --- a/release/conf/cdmi-base-config-sample.xml +++ b/release/conf/cdmi-base-config-sample.xml @@ -1,10 +1,10 @@ - + - + diff --git a/release/start-driver.sh b/release/start-driver.sh index 0f991788..33d3a71d 100644 --- a/release/start-driver.sh +++ b/release/start-driver.sh @@ -22,7 +22,7 @@ SERVICE_NAME=driver VERSION=`cat VERSION` -OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-core_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}" +OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-util_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}" OSGI_CONSOLE_PORT=18089