From 6d259d2ac19fce4c9e80a0ee5dc640ae79112670 Mon Sep 17 00:00:00 2001 From: xiaoerlyl Date: Wed, 17 Dec 2014 13:35:29 +0800 Subject: [PATCH 1/3] if catch 409 error code when delete a unempty caontainer, will clean up the container and try delete it again --- .../com/intel/cosbench/api/auth/AuthAPI.java | 4 + .../cosbench/api/storage/NoneStorage.java | 1 + .../cosbench/driver/agent/WorkAgent.java | 1 - .../cosbench/driver/model/MissionContext.java | 31 +++++- .../driver/operator/AbstractOperator.java | 7 +- .../cosbench/driver/operator/Cleaner.java | 95 +++++++++++++++---- .../driver/service/COSBDriverService.java | 8 +- .../cosbench/client/swift/SwiftClient.java | 1 - release/conf/controller.conf | 2 +- release/conf/driver.conf | 2 +- 10 files changed, 121 insertions(+), 31 deletions(-) diff --git a/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java b/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java index d858ec06..d7228c1e 100644 --- a/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java +++ b/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java @@ -38,6 +38,10 @@ public interface AuthAPI { * from execution engine. */ public void init(Config config, Logger logger); + + /** + * Initializes a new HttpClient for WorkAgent to relogin. + */ public void init(); /** diff --git a/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java b/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java index ea415403..392e48e7 100644 --- a/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java +++ b/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java @@ -61,6 +61,7 @@ public void init(Config config, Logger logger) { @Override public void setAuthContext(AuthContext info) { + setAuthFlag(true); /* empty */ } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java index a624e96d..eebd02a2 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java @@ -295,7 +295,6 @@ public void reLogin() { workerContext.getAuthApi().init(); authContext = workerContext.getAuthApi().login(); workerContext.getStorageApi().setAuthContext(authContext); - workerContext.getStorageApi().setAuthFlag(true); synchronized (AuthCachePool.getInstance()) { AuthCachePool.getInstance().put(authContext.getID(), authContext); } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java index e214a0d1..e54c80fe 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java @@ -22,6 +22,7 @@ import com.intel.cosbench.bench.*; import com.intel.cosbench.config.*; +import com.intel.cosbench.config.common.KVConfigParser; import com.intel.cosbench.driver.util.OperationPicker; import com.intel.cosbench.log.LogManager; import com.intel.cosbench.model.*; @@ -41,7 +42,7 @@ public class MissionContext implements MissionInfo { private StateRegistry stateHistory = new StateRegistry(); private transient XmlConfig config; private transient volatile Future future; - + private Mission mission; private LogManager logManager; private ErrorStatistics errorStatistics; @@ -54,7 +55,10 @@ public class MissionContext implements MissionInfo { private volatile Report report = null; // will be merged from worker reports private transient List listeners = new ArrayList(); - + + private static final String GENERATE_HISTOGRAM_KEY = "histogram"; + private static final boolean DEFAULT_GENERATE_HISTOGRAM = true; + public MissionContext() { errorStatistics = new ErrorStatistics(); } @@ -95,7 +99,7 @@ private void fireMissionStopped() { for (MissionListener listener : listeners) listener.missionStopped(this); } - + private Report mergeReport() { ReportMerger merger = new ReportMerger(); for (WorkerContext worker : workerRegistry) @@ -109,6 +113,27 @@ private Report mergeReport() { return report; } +/* private Report mergeReport() { + ReportMerger merger = new ReportMerger(); + for (WorkerContext worker : workerRegistry) + merger.add(worker.getReport()); + Report report = merger.merge(); + Config missionConfig = KVConfigParser.parse(mission.getConfig()); + boolean histogram = missionConfig.getBoolean(GENERATE_HISTOGRAM_KEY, DEFAULT_GENERATE_HISTOGRAM); + if(histogram) { + generateHistogram(report); + } + return report; + } + + private void generateHistogram(Report report) { + OperatorRegistry registry = operatorRegistry; + for (Metrics metrics : report) { + OperatorContext op = registry.getOperator(metrics.getOpId()); + metrics.setLatency(Histogram.convert(op.getCounter())); + } + } +*/ @Override public StateInfo[] getStateHistory() { return stateHistory.getAllStates(); diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java index b1af4b6a..eb9c9330 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java @@ -17,11 +17,11 @@ package com.intel.cosbench.driver.operator; + import java.util.HashMap; -import com.intel.cosbench.config.Config; -import com.intel.cosbench.api.storage.StorageException; import com.intel.cosbench.bench.ErrorStatistics; +import com.intel.cosbench.config.Config; import com.intel.cosbench.log.LogFactory; import com.intel.cosbench.log.Logger; @@ -136,9 +136,8 @@ public static void isUnauthorizedException(Exception e, Session session) { LOGGER.debug("catch 401 error from storage backend, set auth flag to false"); } }catch(NumberFormatException ne) { - ne.printStackTrace(); // mask ignore + ne.printStackTrace();// mask ignore } } - } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java index cd055613..713206f8 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java @@ -19,8 +19,13 @@ import static com.intel.cosbench.driver.operator.Deleter.doDelete; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import com.intel.cosbench.api.storage.StorageException; @@ -94,24 +99,82 @@ protected void operate(int idx, int all, Session session) { session.getListener().onOperationCompleted(result); } - public static void doDispose(String conName, Config config, Session session) { + public void doDispose(String conName, Config config, Session session) { if (Thread.interrupted()) throw new AbortedException(); - - try { - session.getApi().deleteContainer(conName, config); - } catch (StorageInterruptedException sie) { - throw new AbortedException(); - } catch (StorageException se) { - String msg = "Error deleting container " + conName; - doLogWarn(session.getLogger(), msg); - // ignored - } catch (Exception e) { - doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e); - throw new AgentException(); // mark error - } - - /* no sample is provided for this operation */ + + boolean isEmpty = true; + boolean tryAgain = false; + do { + if(isEmpty) { + try { + session.getApi().deleteContainer(conName, config); + isEmpty = true; + tryAgain = false; + } catch (StorageInterruptedException sie) { + throw new AbortedException(); + } catch (StorageException se) { + String msg = "Error deleting container " + conName; + doLogErr(session.getLogger(), msg,se); + if(isConflictException(session,se)) { + isEmpty = false; + tryAgain = true; + }else{ + tryAgain = false; + } + } catch (Exception e) { + doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e); + throw new AgentException(); // mark error + } + }else { + try{ + for(String objName: getObjectsList(conName, config, session)) { + doDelete(conName, objName, config, session, this); + } + }catch(StorageException se) { + doLogErr(session.getLogger(), "fail to get : "+conName+" objects list ",se); + tryAgain = false; + }catch (IOException ioe) { + doLogErr(session.getLogger(), "fail to convert objects stream to string",ioe); + tryAgain = false; + }catch (Exception e) { + doLogErr(session.getLogger(), "unexpected error",e); + tryAgain = false; + } + isEmpty = true; + tryAgain = true; + } + }while(tryAgain == true); + + } + + private static String[] getObjectsList(String conName, Config config, Session session) throws IOException { + String[] objects = {}; + StringWriter stringWriter = new StringWriter(); + try { + IOUtils.copy(session.getApi().getList(conName, "", config), stringWriter); + }catch(StorageException se) { + throw se; + }catch (IOException e) { + throw e; + } + String objectString = stringWriter.toString(); + objects = objectString.split("\n"); + return objects; + } + + private static boolean isConflictException(Session session, Exception e) { + if(e != null && e.getMessage() != null) + try{ + if(409 == Integer.valueOf(e.getMessage().substring(9, 12))){ + doLogDebug(session.getLogger(),"catch 409 error, will clean up the unempty container and try again"); + return true; + } + }catch(NumberFormatException ne) { + ne.printStackTrace(); // mask ignore + return false; + } + return false; } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java index 330de980..1e596d1c 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java @@ -43,16 +43,16 @@ class COSBDriverService implements DriverService, MissionListener { private static final Logger LOGGER = LogFactory.getSystemLogger(); - + private DriverContext context; private Map handlers; - + private AuthAPIService authAPIs; private StorageAPIService storageAPIs; private ExecutorService executor; private MissionRepository memRepo = new RAMMissionRepository(); - + public COSBDriverService() { /* empty */ } @@ -74,7 +74,7 @@ public void init() { handlers = Collections.synchronizedMap(handlers); executor = Executors.newCachedThreadPool(); } - + @Override public String submit(XmlConfig config) { LOGGER.debug("submitting mission ... "); diff --git a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java index 7c7cf66b..02ec1c7a 100644 --- a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java +++ b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java @@ -27,7 +27,6 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.*; import org.apache.http.entity.*; - import com.intel.cosbench.client.http.HttpClientUtil; import com.intel.cosbench.log.*; diff --git a/release/conf/controller.conf b/release/conf/controller.conf index b8bab9a4..313b6a2c 100644 --- a/release/conf/controller.conf +++ b/release/conf/controller.conf @@ -1,6 +1,6 @@ [controller] drivers = 1 -log_level = INFO +log_level = DEBUG log_file = log/system.log archive_dir = archive diff --git a/release/conf/driver.conf b/release/conf/driver.conf index 7338ffe8..cd1d83a7 100644 --- a/release/conf/driver.conf +++ b/release/conf/driver.conf @@ -1,2 +1,2 @@ [driver] -log_level = INFO \ No newline at end of file +log_level = DEBUG \ No newline at end of file From fff77bff514700ea862f8118bd16a3fb91cedf32 Mon Sep 17 00:00:00 2001 From: xiaoerlyl Date: Fri, 19 Dec 2014 11:09:39 +0800 Subject: [PATCH 2/3] resolve 409 --- .../cosbench/driver/operator/Cleaner.java | 96 +++++++++++++++---- release/conf/controller.conf | 2 +- release/conf/driver.conf | 2 +- 3 files changed, 81 insertions(+), 19 deletions(-) diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java index 542b50cb..713206f8 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java @@ -19,8 +19,13 @@ import static com.intel.cosbench.driver.operator.Deleter.doDelete; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import com.intel.cosbench.api.storage.StorageException; @@ -94,25 +99,82 @@ protected void operate(int idx, int all, Session session) { session.getListener().onOperationCompleted(result); } - public static void doDispose(String conName, Config config, Session session) { + public void doDispose(String conName, Config config, Session session) { if (Thread.interrupted()) throw new AbortedException(); - - try { - session.getApi().deleteContainer(conName, config); - } catch (StorageInterruptedException sie) { - doLogErr(session.getLogger(), sie.getMessage(), sie); - throw new AbortedException(); - } catch (StorageException se) { - String msg = "Error deleting container " + conName; - doLogWarn(session.getLogger(), msg); - // ignored - } catch (Exception e) { - doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e); - throw new AgentException(); // mark error - } - - /* no sample is provided for this operation */ + + boolean isEmpty = true; + boolean tryAgain = false; + do { + if(isEmpty) { + try { + session.getApi().deleteContainer(conName, config); + isEmpty = true; + tryAgain = false; + } catch (StorageInterruptedException sie) { + throw new AbortedException(); + } catch (StorageException se) { + String msg = "Error deleting container " + conName; + doLogErr(session.getLogger(), msg,se); + if(isConflictException(session,se)) { + isEmpty = false; + tryAgain = true; + }else{ + tryAgain = false; + } + } catch (Exception e) { + doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e); + throw new AgentException(); // mark error + } + }else { + try{ + for(String objName: getObjectsList(conName, config, session)) { + doDelete(conName, objName, config, session, this); + } + }catch(StorageException se) { + doLogErr(session.getLogger(), "fail to get : "+conName+" objects list ",se); + tryAgain = false; + }catch (IOException ioe) { + doLogErr(session.getLogger(), "fail to convert objects stream to string",ioe); + tryAgain = false; + }catch (Exception e) { + doLogErr(session.getLogger(), "unexpected error",e); + tryAgain = false; + } + isEmpty = true; + tryAgain = true; + } + }while(tryAgain == true); + + } + + private static String[] getObjectsList(String conName, Config config, Session session) throws IOException { + String[] objects = {}; + StringWriter stringWriter = new StringWriter(); + try { + IOUtils.copy(session.getApi().getList(conName, "", config), stringWriter); + }catch(StorageException se) { + throw se; + }catch (IOException e) { + throw e; + } + String objectString = stringWriter.toString(); + objects = objectString.split("\n"); + return objects; + } + + private static boolean isConflictException(Session session, Exception e) { + if(e != null && e.getMessage() != null) + try{ + if(409 == Integer.valueOf(e.getMessage().substring(9, 12))){ + doLogDebug(session.getLogger(),"catch 409 error, will clean up the unempty container and try again"); + return true; + } + }catch(NumberFormatException ne) { + ne.printStackTrace(); // mask ignore + return false; + } + return false; } } diff --git a/release/conf/controller.conf b/release/conf/controller.conf index 313b6a2c..b8bab9a4 100644 --- a/release/conf/controller.conf +++ b/release/conf/controller.conf @@ -1,6 +1,6 @@ [controller] drivers = 1 -log_level = DEBUG +log_level = INFO log_file = log/system.log archive_dir = archive diff --git a/release/conf/driver.conf b/release/conf/driver.conf index cd1d83a7..7338ffe8 100644 --- a/release/conf/driver.conf +++ b/release/conf/driver.conf @@ -1,2 +1,2 @@ [driver] -log_level = DEBUG \ No newline at end of file +log_level = INFO \ No newline at end of file From d71f05fdde85d79a50285d91207b03432958ecb8 Mon Sep 17 00:00:00 2001 From: xiaoerlyl Date: Fri, 19 Dec 2014 12:35:32 +0800 Subject: [PATCH 3/3] optional configuration for generating histogram --- .../cosbench/driver/model/MissionContext.java | 22 ------------------- release/conf/controller.conf | 2 +- release/conf/driver.conf | 2 +- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java index 7386ed6d..3fa31af0 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java @@ -120,28 +120,6 @@ private void generateHistogram(Report report) { metrics.setLatency(Histogram.convert(op.getCounter())); } } - -/* private Report mergeReport() { - ReportMerger merger = new ReportMerger(); - for (WorkerContext worker : workerRegistry) - merger.add(worker.getReport()); - Report report = merger.merge(); - Config missionConfig = KVConfigParser.parse(mission.getConfig()); - boolean histogram = missionConfig.getBoolean(GENERATE_HISTOGRAM_KEY, DEFAULT_GENERATE_HISTOGRAM); - if(histogram) { - generateHistogram(report); - } - return report; - } - - private void generateHistogram(Report report) { - OperatorRegistry registry = operatorRegistry; - for (Metrics metrics : report) { - OperatorContext op = registry.getOperator(metrics.getOpId()); - metrics.setLatency(Histogram.convert(op.getCounter())); - } - } -*/ @Override public StateInfo[] getStateHistory() { return stateHistory.getAllStates(); diff --git a/release/conf/controller.conf b/release/conf/controller.conf index 313b6a2c..b8bab9a4 100644 --- a/release/conf/controller.conf +++ b/release/conf/controller.conf @@ -1,6 +1,6 @@ [controller] drivers = 1 -log_level = DEBUG +log_level = INFO log_file = log/system.log archive_dir = archive diff --git a/release/conf/driver.conf b/release/conf/driver.conf index cd1d83a7..7338ffe8 100644 --- a/release/conf/driver.conf +++ b/release/conf/driver.conf @@ -1,2 +1,2 @@ [driver] -log_level = DEBUG \ No newline at end of file +log_level = INFO \ No newline at end of file