Merge pull request #1 from intel-cloud/master

0.4.0.1

CHANGELOG

@@ -1,4 +1,4 @@
-  COSBench 0.4.0.b2 (14-ww17.1)
+  COSBench 0.4.0.0 (14-ww21.5)
 ------------------------------------
 . #57: time synchronization
 . #96: Creating Web UI facility for generating different workload configuration files
@@ -8,10 +8,6 @@
 . #167: storage policy supporting for openstack swift
 . #37: support CDMI interface
 . #139: NullPointerException when a mission is aborted
-
-
-  COSBench 0.4.0.b1 (14-ww11.4)
-------------------------------------
 . #166: show driver alive state on controller index page
 . #158: Extend "histogram" selector to support "open ranges"
 . #165: enable Open range for object size in uniform and histogram
@@ -21,10 +17,6 @@
 . #143: at loading archived workloads, even there are 100 workloads, the UI shows only 98.
 . #140: duplicated workload id generated 
 . #151: avoid being challenged for each request
-
-
-  COSBench 0.4.0.a1 (14-ww1.4)
-------------------------------------
 . #137: when there are large # of archive jobs, exception will raise from archive list on web portal.
 . #139: NullPointerException when a mission is aborted
 . support http BASIC and DIGEST authentication

README.md

@@ -64,4 +64,11 @@ Mailing list: (http://cosbench.1094679.n5.nabble.com/)
 *Other names and brands may be claimed as the property of others.
 
 
+Other related projects
+----------------------
+COSBench-Workload-Generator: (https://github.com/giteshnandre/COSBench-Workload-Generator)
+
+COSBench-Plot: (https://github.com/icclab/cosbench-plot)
+
+
 = END =

TODO.md

@@ -2,13 +2,13 @@ TODO
 ----
 COSBench is an ongoing project, in next six months, we plan to make below improvements:
 
-1. more storage interface support, e.g. s3 and cdmi
+1. more storage interface support, list is TBD.
 
-2. async http request
+2. authentication caching
 
-3. usability enhancement, e.g. xml validator, error handling.
+3. LIST operation
 
-4. document improvement
+4. workload packages shipped
 
 
 If you have any other suggestions or you want to work with us together. please contact us at yaguang.wang@intel.com.

dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java

@@ -41,7 +41,7 @@ public class AmpliClient {
 
     private HttpClient client = null;
     /* current operation */
-    private volatile HttpUriRequest method;
+    private volatile HttpUriRequest request;
     private int port;
     private String host;
     private String nsRoot;
@@ -54,20 +54,20 @@ public AmpliClient(HttpClient client, String host, int port, String nsRoot) {
     }
 
     public void dispose() {
-    	method = null;
+    	request = null;
         HttpClientUtil.disposeHttpClient(client);
     }
 
     public void abort() {
-        if (method != null)
-            method.abort();
-        method = null;
+        if (request != null)
+            request.abort();
+        request = null;
     }
 
     public boolean login() throws IOException, HttpException {
         String storageUrl = "http://" + this.host + ":" + this.port;
 
-        method = HttpClientUtil.makeHttpHead(storageUrl);
+        HttpHead method = HttpClientUtil.makeHttpHead(storageUrl);
         HttpResponse response = null;
         try {
             response = client.execute(method);
@@ -89,14 +89,15 @@ public String StoreObject(String sourceFilename, String ampliNamespace,
             AmpliException {
         File file = new File(sourceFilename);
 
+        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
             method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace)
                     + "/" + HttpClientUtil.encodeURL(ampliFilename));
 
-            ((HttpPut)method).setEntity(new FileEntity(file, "application/octet-stream"));
+            method.setEntity(new FileEntity(file, "application/octet-stream"));
 
             response = client.execute(method);
 
@@ -126,6 +127,7 @@ public String StoreObject(String sourceFilename, String ampliNamespace,
     public String StoreStreamedObject(InputStream stream, long length,
             String ampliNamespace, String ampliFilename) throws IOException,
             HttpException, AmpliException {
+        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -141,7 +143,7 @@ public String StoreStreamedObject(InputStream stream, long length,
                 entity.setChunked(false);
             }
 
-            ((HttpPut)method).setEntity(entity);
+            method.setEntity(entity);
 
             response = client.execute(method);
 
@@ -189,6 +191,7 @@ public String StoreObject(byte[] data, String ampliNamespace,
             String ampliFilename) throws IOException, HttpException,
             AmpliException {
         // int len = data.length;
+        HttpPut method = null;
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
         method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace) + "/"
                 + HttpClientUtil.encodeURL(ampliFilename));
@@ -197,7 +200,7 @@ public String StoreObject(byte[] data, String ampliNamespace,
 
         HttpResponse response = null;
         try {
-            ((HttpPut)method).setEntity(new ByteArrayEntity(data));
+            method.setEntity(new ByteArrayEntity(data));
             response = client.execute(method);
             if (response.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED) {
                 return response.getFirstHeader("ETag").getValue();
@@ -219,7 +222,7 @@ public byte[] getObject(String namespace, String objName)
             throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName));
 
         HttpResponse response = null;
@@ -246,7 +249,7 @@ public InputStream getObjectAsStream(String namespace, String objName)
             throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName));
 
         HttpResponse response = null;
@@ -269,6 +272,7 @@ public InputStream getObjectAsStream(String namespace, String objName)
     public boolean deleteObject(String ampliNamespace, String name)
             throws HttpException, IOException, AmpliException {
 
+        HttpDelete method = null;
         HttpResponse response = null;
 
         try {
@@ -298,6 +302,7 @@ public boolean deleteObject(String ampliNamespace, String name)
     public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException,
             IOException, AmpliException {
 
+        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -331,6 +336,7 @@ public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException,
     public AmpliPolicy getPolicy(String policyId) throws HttpException,
             IOException, AmpliException {
 
+        HttpGet method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -360,13 +366,13 @@ public AmpliPolicy getPolicy(String policyId) throws HttpException,
 
     public AmpliNamespace createNamespace(AmpliNamespace namespace)
             throws HttpException, IOException, AmpliException {
-
-    	HttpResponse response = null;
+        HttpPut method = null;
+        HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
 
-            method = HttpClientUtil.makeHttpGet(storageUrl);
+            method = new HttpPut(storageUrl);
 
             method.setHeader("Content-Type", "text/plain");
 
@@ -399,7 +405,7 @@ public String createNamespace(String namespace, String policy_id)
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
 
-            method = HttpClientUtil.makeHttpPut(storageUrl);
+            method = new HttpPut(storageUrl);
 
             method.setHeader("Content-Type", "text/plain");
 
@@ -432,6 +438,7 @@ public String createNamespace(String namespace, String policy_id)
     public AmpliNamespace getNamespace(String name) throws HttpException,
             IOException, AmpliException {
 
+        HttpGet method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -462,6 +469,7 @@ public AmpliNamespace getNamespace(String name) throws HttpException,
     public boolean isNamespaceExisted(String name) throws HttpException,
             IOException, AmpliException {
 
+        HttpHead method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -489,7 +497,7 @@ public boolean isNamespaceExisted(String name) throws HttpException,
 
     public boolean deleteNamespace(String namespace) throws HttpException,
             IOException, AmpliException {
-
+        HttpDelete method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -517,7 +525,7 @@ public Map<String, String> getObjectMetadata(String namespace,
             String objName) throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName) + "?meta=http");
 
         HttpResponse response = null;

dev/cosbench-controller-web/WEB-INF/freemarker/advanced-config.ftl

@@ -13,13 +13,15 @@
   <h3>Workload Matrix Configuration</h3>
     <h5>You can configure workload matrix from here. You can generate or submit workloads directly. With 'Generate Workload File/s' option,
         files will be generated and placed at 'workloads' directory inside COSBench installation directory. No workload files will be generated when
-        'Submit Workload/s' option is chosen and workloads will be submitted directly.</h5>
+        'Submit Workload/s' option is chosen and workloads will be submitted directly.
+        <br>(Note: * marked fields are mandatory.)</br>
+    </h5>
   <div>
     <form action="advanced-config-workload.do" method="post" class="content" >
 	  <#if error?? >
         <p><span class="error"><strong>Note</strong>: ${error}!</span></p>
       </#if>
-      	<label for="workload.matrix.name" >Workload Matrix Name:</label>
+      	<label for="workload.matrix.name" ><strong>*</strong>Workload Matrix Name:</label>
         <input id="workload.matrix.name" name="workload.matrix.name" size="15" maxlength="30"/>
       	<div id="workload-common-configuration" class="a1">
       	<div id="workload.as" class="a2">
@@ -103,7 +105,7 @@
 				
 			
 			<input id="workload-checkbox" type="checkbox" name="workload.matrix.checked" checked="checked" onClick="toggleDiv(this.nextElementSibling.nextElementSibling.nextElementSibling);" style="float:left">
-			<label for="workload-checkbox" class="a2">Workload Name:</label>
+			<label for="workload-checkbox" class="a2">*Workload Name:</label>
 			<input name="workload.name" size="15" maxlength="30"/>
 			<div name="workload.matrix" id="workload.matrix" class="a2" >
 					<input type="hidden" id="workload-number" name="workload-number" value="0">

dev/cosbench-controller/src/com/intel/cosbench/controller/service/WorkloadProcessor.java

@@ -290,13 +290,18 @@ public void cancel() {
     private void cancelWorkload() {
         String id = workloadContext.getId();
         LOGGER.info("begin to cancel workload {}", id);
-        executor.shutdownNow();
+        executor.shutdown();
         if (Thread.interrupted())
-            LOGGER.warn("get cancelled when canceling workload");
+            LOGGER.warn("get cancelled when canceling workload {}", id);
         try {
-            executor.awaitTermination(5, TimeUnit.SECONDS);
+        	if (!executor.awaitTermination(5, TimeUnit.SECONDS)
+        			&& !executor.awaitTermination(5, TimeUnit.SECONDS))
+				executor.shutdownNow();
+            if (!awaitTermination(5) && !awaitTermination(10) && !awaitTermination(30))
+            	LOGGER.warn("get cancelled when canceling workload {}", id);
         } catch (InterruptedException e) {
-            LOGGER.warn("get cancelled when canceling workload");
+            executor.shutdownNow();
+            Thread.currentThread().interrupt();
         }
         if (!executor.isTerminated())
             LOGGER.warn("fail to cancel current stage for workload {}", id);
@@ -309,5 +314,17 @@ private void cancelWorkload() {
         workloadContext.setState(CANCELLED);
         LOGGER.info("successfully cancelled workload {}", id);
     }
+
+    private boolean awaitTermination(int seconds) {
+        try {
+            if (!executor.isTerminated()) {
+                LOGGER.info("wait {} seconds for workload to cancel ...", seconds);
+                executor.awaitTermination(seconds, TimeUnit.SECONDS);
+            }
+        } catch (InterruptedException e) {
+            LOGGER.debug("get cancelled when canceling workload");
+        }
+        return executor.isTerminated();
+    }
 
 }

dev/cosbench-controller/src/com/intel/cosbench/controller/tasklet/Aborter.java

@@ -19,6 +19,8 @@
 
 import static com.intel.cosbench.model.TaskState.*;
 
+import com.intel.cosbench.bench.Metrics;
+import com.intel.cosbench.bench.Report;
 import com.intel.cosbench.controller.model.TaskContext;
 import com.intel.cosbench.protocol.AbortResponse;
 
@@ -61,6 +63,10 @@ private void executeAbort() {
 
     @Override
     protected void handleResponse(AbortResponse response) {
+    	Report report = new Report();
+        for (Metrics metrics : response.getReport())
+            report.addMetrics(metrics);
+        context.setReport(report);
         context.setLog(response.getDriverLog());
     }
 

dev/cosbench-core/src/com/intel/cosbench/protocol/AbortResponse.java

@@ -17,6 +17,10 @@
 
 package com.intel.cosbench.protocol;
 
+import java.util.List;
+
+import com.intel.cosbench.bench.Metrics;
+
 
 /**
  * The response to get log from driver when aborted.
@@ -27,6 +31,7 @@
 public class AbortResponse extends Response {
 
     private String driverLog; /* driver log */
+    private List<Metrics> report; /* metrics report */
 
     public AbortResponse() {
         /* empty */
@@ -39,5 +44,13 @@ public String getDriverLog() {
     public void setDriverLog(String driverLog) {
         this.driverLog = driverLog;
     }
+
+    public List<Metrics> getReport() {
+        return report;
+    }
+
+    public void setReport(List<Metrics> report) {
+        this.report = report;
+    }
 
 }

dev/cosbench-driver-web/src/com/intel/cosbench/driver/handler/AbortHandler.java

@@ -18,7 +18,9 @@
 package com.intel.cosbench.driver.handler;
 
 import java.io.IOException;
+import java.util.Arrays;
 
+import com.intel.cosbench.bench.Report;
 import com.intel.cosbench.model.MissionInfo;
 import com.intel.cosbench.protocol.*;
 
@@ -33,6 +35,8 @@ protected Response process(MissionInfo info) {
 
     private Response getResponse(MissionInfo info) {
         AbortResponse response = new AbortResponse();
+        Report report = info.getReport();
+        response.setReport(Arrays.asList(report.getAllMetrics()));
         String log = null;
         try {
             log = info.getLogManager().getLogAsString();

dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WatchDog.java

@@ -64,7 +64,7 @@ public void run() {
             secs -= 5;
         }
         if (!cancel) {
-//            workerContext.getStorageApi().abort();
+            workerContext.getStorageApi().abort();
         	workerContext.disposeRuntime();
 
             LOGGER.debug("work agent {} has been alerted for timeout", idx);