From 0a52a12c67773393e823d6539983b6b538802945 Mon Sep 17 00:00:00 2001 From: tobiast Date: Thu, 27 Apr 2017 11:31:15 -0700 Subject: [PATCH 01/34] Caching to avoid reparsing SQL text --- .gitignore | 1 + build.gradle | 5 +- pom.xml | 7 ++ .../jdbc/SQLServerPreparedStatement.java | 92 +++++++++++++++++-- 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index fad40fcb3..acb9b8d91 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ local.properties .classpath .vscode/ .settings/ +.gradle/ .loadpath # External tool builders diff --git a/build.gradle b/build.gradle index 55792f0a3..38e407d03 100644 --- a/build.gradle +++ b/build.gradle @@ -67,8 +67,9 @@ repositories { dependencies { compile 'com.microsoft.azure:azure-keyvault:0.9.7', - 'com.microsoft.azure:adal4j:1.1.3' - + 'com.microsoft.azure:adal4j:1.1.3', + 'com.google.guava:guava:19.0' + testCompile 'junit:junit:4.12', 'org.junit.platform:junit-platform-console:1.0.0-M3', 'org.junit.platform:junit-platform-commons:1.0.0-M3', diff --git a/pom.xml b/pom.xml index 8dfd2d078..dfcc503e0 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,13 @@ true + + com.google.guava + guava + 19.0 + false + + junit diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index b1dff7c4d..923914f4e 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -28,6 +28,9 @@ import java.util.Vector; import java.util.logging.Level; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.Cache; + /** * SQLServerPreparedStatement provides JDBC prepared statement functionality. SQLServerPreparedStatement provides methods for the user to supply * parameters as any native Java type and many Java object types. @@ -98,6 +101,48 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS */ private boolean encryptionMetadataIsRetrieved = false; + /** Size of the prepared statement meta data cache */ + static final private int preparedStatementMetadataSQLCacheSize = 100; + + /** Cache of prepared statement meta data */ + static private Cache preparedStatementSQLMetadataCache; + static { + preparedStatementSQLMetadataCache = CacheBuilder.newBuilder() + .maximumSize(preparedStatementMetadataSQLCacheSize) + .build(); + } + + /** + * Used to keep track of an individual handle ready for un-prepare. + */ + private final class PreparedStatementMetadataSQLCacheItem { + String preparedSQLText; + int parameterCount; + String procedureName; + boolean bReturnValueSyntax; + + PreparedStatementMetadataSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax){ + this.preparedSQLText = preparedSQLText; + this.parameterCount = parameterCount; + this.procedureName = procedureName; + this.bReturnValueSyntax = bReturnValueSyntax; + } + } + + /** Get prepared statement cache entry if exists */ + public PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql){ + return preparedStatementSQLMetadataCache.getIfPresent(initialSql); + } + + /** Cache entry for this prepared statement */ + public PreparedStatementMetadataSQLCacheItem metadataSQLCacheItem; + + /** Add cache entry for prepared statement metadata*/ + public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem){ + + preparedStatementSQLMetadataCache.put(initialSql, newItem); + } + // Internal function used in tracing String getClassNameInternal() { return "SQLServerPreparedStatement"; @@ -128,13 +173,34 @@ String getClassNameInternal() { stmtPoolable = true; sqlCommand = sql; - JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); - sql = translator.translate(sql); - procedureName = translator.getProcedureName(); // may return null - bReturnValueSyntax = translator.hasReturnValueSyntax(); - - userSQL = sql; - initParams(userSQL); + // Save original SQL statement. + sqlCommand = sql; + + // Check for cached SQL metadata. + PreparedStatementMetadataSQLCacheItem cacheItem = getCachedPreparedStatementSQLMetadata(sql); + + // No cached meta data found, parse. + if(null == cacheItem) { + JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); + + userSQL = translator.translate(sql); + procedureName = translator.getProcedureName(); // may return null + bReturnValueSyntax = translator.hasReturnValueSyntax(); + + // Save processed SQL statement. + initParams(userSQL); + + // Cache this entry. + cacheItem = new PreparedStatementMetadataSQLCacheItem(userSQL, inOutParam.length, procedureName, bReturnValueSyntax); + cachePreparedStatementSQLMetaData(sqlCommand/*original command as key*/, cacheItem); + } + else { + // Retrieve from cache item. + procedureName = cacheItem.procedureName; + bReturnValueSyntax = cacheItem.bReturnValueSyntax; + userSQL = cacheItem.preparedSQLText; + initParams(cacheItem.parameterCount); + } } /** @@ -217,12 +283,11 @@ final void closeInternal() { } /** - * Intialize the statement parameters. + * Find and intialize the statement parameters. * * @param sql */ /* L0 */ final void initParams(String sql) { - encryptionMetadataIsRetrieved = false; int nParams = 0; // Figure out the expected number of parameters by counting the @@ -231,6 +296,15 @@ final void closeInternal() { while ((offset = ParameterUtils.scanSQLForChar('?', sql, ++offset)) < sql.length()) ++nParams; + initParams(nParams); + } + + /** + * Intialize the statement parameters. + * + * @param sql + */ + /* L0 */ final void initParams(int nParams) { inOutParam = new Parameter[nParams]; for (int i = 0; i < nParams; i++) { inOutParam[i] = new Parameter(Util.shouldHonorAEForParameters(stmtColumnEncriptionSetting, connection)); From c886dcfe84fa63c113bbef68e1c1ae147ce717e6 Mon Sep 17 00:00:00 2001 From: tobiast Date: Thu, 27 Apr 2017 11:40:07 -0700 Subject: [PATCH 02/34] Minor clean-up for SQL text caching. --- .../sqlserver/jdbc/SQLServerPreparedStatement.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 923914f4e..352d6dbad 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -121,7 +121,7 @@ private final class PreparedStatementMetadataSQLCacheItem { String procedureName; boolean bReturnValueSyntax; - PreparedStatementMetadataSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax){ + PreparedStatementMetadataSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { this.preparedSQLText = preparedSQLText; this.parameterCount = parameterCount; this.procedureName = procedureName; @@ -130,15 +130,12 @@ private final class PreparedStatementMetadataSQLCacheItem { } /** Get prepared statement cache entry if exists */ - public PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql){ + public PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql) { return preparedStatementSQLMetadataCache.getIfPresent(initialSql); } - /** Cache entry for this prepared statement */ - public PreparedStatementMetadataSQLCacheItem metadataSQLCacheItem; - /** Add cache entry for prepared statement metadata*/ - public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem){ + public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { preparedStatementSQLMetadataCache.put(initialSql, newItem); } @@ -286,6 +283,7 @@ final void closeInternal() { * Find and intialize the statement parameters. * * @param sql + * SQL text to parse for number of parameters to intialize. */ /* L0 */ final void initParams(String sql) { int nParams = 0; @@ -302,7 +300,8 @@ final void closeInternal() { /** * Intialize the statement parameters. * - * @param sql + * @param nParams + * Number of parameters to Intialize. */ /* L0 */ final void initParams(int nParams) { inOutParam = new Parameter[nParams]; From ed144a431d51b7be4bc511c1b399ed64381ed160 Mon Sep 17 00:00:00 2001 From: tobiast Date: Thu, 27 Apr 2017 12:42:48 -0700 Subject: [PATCH 03/34] Cache methods should be private. --- .../sqlserver/jdbc/SQLServerConnection.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 54c5950c2..f898fc01b 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -56,6 +56,9 @@ import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.Cache; + /** * SQLServerConnection implements a JDBC connection to SQL Server. SQLServerConnections support JDBC connection pooling and may be either physical * JDBC connections or logical JDBC connections. @@ -115,6 +118,33 @@ public class SQLServerConnection implements ISQLServerConnection { private SqlFedAuthToken fedAuthToken = null; + /** + * Used to keep track of an individual handle ready for un-prepare. + */ + private final class PreparedStatementHandle { + String preparedSQLText; + int parameterCount; + String procedureName; + boolean bReturnValueSyntax; + + PreparedStatementHandle(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + this.preparedSQLText = preparedSQLText; + this.parameterCount = parameterCount; + this.procedureName = procedureName; + this.bReturnValueSyntax = bReturnValueSyntax; + } + } + /** Size of the prepared statement meta data cache */ + static final private int preparedStatementHandleCacheSize = 100; + + /** Cache of prepared statement meta data */ + private Cache preparedStatementHandleCache; + static { + preparedStatementHandleCache = CacheBuilder.newBuilder() + .maximumSize(preparedStatementHandleCacheSize) + .build(); + } + SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; } @@ -5436,6 +5466,18 @@ final void handlePreparedStatementDiscardActions(boolean force) { } } } + + + /** Get prepared statement cache entry if exists */ + public PreparedStatementHandle getCachedPreparedStatementHandle(String initialSql) { + return preparedStatementSQLMetadataCache.getIfPresent(initialSql); + } + + /** Add cache entry for prepared statement metadata*/ + public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { + + preparedStatementSQLMetadataCache.put(initialSql, newItem); + } } // Helper class for security manager functions used by SQLServerConnection class. From 418528b012c07c3181312b670116a7773651edba Mon Sep 17 00:00:00 2001 From: tobiast Date: Thu, 27 Apr 2017 12:45:40 -0700 Subject: [PATCH 04/34] Actual clean-up, reverted invalid commit. --- .../sqlserver/jdbc/SQLServerConnection.java | 38 ------------------- .../jdbc/SQLServerPreparedStatement.java | 26 ++++++------- 2 files changed, 13 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index f898fc01b..d266d3d28 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -118,33 +118,6 @@ public class SQLServerConnection implements ISQLServerConnection { private SqlFedAuthToken fedAuthToken = null; - /** - * Used to keep track of an individual handle ready for un-prepare. - */ - private final class PreparedStatementHandle { - String preparedSQLText; - int parameterCount; - String procedureName; - boolean bReturnValueSyntax; - - PreparedStatementHandle(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { - this.preparedSQLText = preparedSQLText; - this.parameterCount = parameterCount; - this.procedureName = procedureName; - this.bReturnValueSyntax = bReturnValueSyntax; - } - } - /** Size of the prepared statement meta data cache */ - static final private int preparedStatementHandleCacheSize = 100; - - /** Cache of prepared statement meta data */ - private Cache preparedStatementHandleCache; - static { - preparedStatementHandleCache = CacheBuilder.newBuilder() - .maximumSize(preparedStatementHandleCacheSize) - .build(); - } - SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; } @@ -5467,17 +5440,6 @@ final void handlePreparedStatementDiscardActions(boolean force) { } } - - /** Get prepared statement cache entry if exists */ - public PreparedStatementHandle getCachedPreparedStatementHandle(String initialSql) { - return preparedStatementSQLMetadataCache.getIfPresent(initialSql); - } - - /** Add cache entry for prepared statement metadata*/ - public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { - - preparedStatementSQLMetadataCache.put(initialSql, newItem); - } } // Helper class for security manager functions used by SQLServerConnection class. diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 352d6dbad..939f4a130 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -101,17 +101,6 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS */ private boolean encryptionMetadataIsRetrieved = false; - /** Size of the prepared statement meta data cache */ - static final private int preparedStatementMetadataSQLCacheSize = 100; - - /** Cache of prepared statement meta data */ - static private Cache preparedStatementSQLMetadataCache; - static { - preparedStatementSQLMetadataCache = CacheBuilder.newBuilder() - .maximumSize(preparedStatementMetadataSQLCacheSize) - .build(); - } - /** * Used to keep track of an individual handle ready for un-prepare. */ @@ -129,13 +118,24 @@ private final class PreparedStatementMetadataSQLCacheItem { } } + /** Size of the prepared statement meta data cache */ + static final private int preparedStatementMetadataSQLCacheSize = 100; + + /** Cache of prepared statement meta data */ + static private Cache preparedStatementSQLMetadataCache; + static { + preparedStatementSQLMetadataCache = CacheBuilder.newBuilder() + .maximumSize(preparedStatementMetadataSQLCacheSize) + .build(); + } + /** Get prepared statement cache entry if exists */ - public PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql) { + private PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql) { return preparedStatementSQLMetadataCache.getIfPresent(initialSql); } /** Add cache entry for prepared statement metadata*/ - public void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { + private void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { preparedStatementSQLMetadataCache.put(initialSql, newItem); } From 4202e45e34f05b9fbb8fd37aeb99a8214459bd88 Mon Sep 17 00:00:00 2001 From: tobiast Date: Sun, 30 Apr 2017 01:20:15 -0500 Subject: [PATCH 05/34] Added prototype for prepared statment handle cache. --- .../sqlserver/jdbc/SQLServerConnection.java | 98 +++++++++++++- .../jdbc/SQLServerPreparedStatement.java | 128 +++++++++++------- .../unit/statement/PreparedStatementTest.java | 6 + 3 files changed, 184 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index d266d3d28..0d4a3c96e 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -56,8 +56,10 @@ import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; -import com.google.common.cache.CacheBuilder; import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; /** * SQLServerConnection implements a JDBC connection to SQL Server. SQLServerConnections support JDBC connection pooling and may be either physical @@ -118,6 +120,55 @@ public class SQLServerConnection implements ISQLServerConnection { private SqlFedAuthToken fedAuthToken = null; + /** + * Used to keep track of an individual handle ready for un-prepare. + */ + final class PreparedStatementHandle { + int handle; + boolean directSql; + SQLServerConnection connection; + private AtomicInteger refCount = new AtomicInteger(1); + + PreparedStatementHandle(int handle, boolean directSql, SQLServerConnection connection) { + this.handle = handle; + this.directSql = directSql; + this.connection = connection; + } + + // Returns false if handle is not re-usable. + boolean incrementRefCountAndVerifyNotInvalidated() { + // If refcount is negative the handle has been killed. + if(0 > this.refCount.getAndIncrement()) { + this.refCount.getAndDecrement(); // Reduce again. + return false; + } + else + return true; + } + + boolean discardIfNotReferenced() { + // If refcount is zero or negative the handle can be killed. + if(1 > this.refCount.getAndDecrement()) { + return true; + } + else { + // In use. + this.refCount.getAndIncrement(); // Return back. + return false; + } + } + + void decrementRefCount() { + this.refCount.decrementAndGet(); + } + } + + /** Size of the prepared statement meta data cache */ + static public int preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 10; + + /** Cache of prepared statement meta data */ + private Cache preparedStatementHandleCache; + SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; } @@ -5438,8 +5489,51 @@ final void handlePreparedStatementDiscardActions(boolean force) { this.discardedPreparedStatementHandleQueueCount.addAndGet(-handlesRemoved); } } - } + } + + + /** Get prepared statement cache entry if exists */ + final PreparedStatementHandle getCachedPreparedStatementHandle(String sql) { + if(null == this.preparedStatementHandleCache) + return null; + + return this.preparedStatementHandleCache.getIfPresent(sql); + } + + // Handle closing handles when removed from cache. + RemovalListener preparedStatementHandleCacheRemovalListener = new RemovalListener() { + public void onRemoval(RemovalNotification removal) { + PreparedStatementHandle handle = removal.getValue(); + // Only discard if not referenced. + if(null != handle && handle.discardIfNotReferenced()) { + handle.connection.enqueuePreparedStatementDiscardItem(handle.handle, handle.directSql); + handle.connection.handlePreparedStatementDiscardActions(false); + } + else if(null != handle) + // Put back in cache. + handle.connection.cachePreparedStatementHandle(removal.getKey(), handle); + } + }; + + /** Add cache entry for prepared statement metadata*/ + final void cachePreparedStatementHandle(String sql, int handle, boolean directSql) { + this.cachePreparedStatementHandle(sql, new PreparedStatementHandle(handle, directSql, this)); + } + + /** Add cache entry for prepared statement metadata*/ + final void cachePreparedStatementHandle(String sql, PreparedStatementHandle handle) { + // Caching turned off? + if(0 >= SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY || null == handle) + return; + + if(null == this.preparedStatementHandleCache) { + preparedStatementHandleCache = CacheBuilder.newBuilder() + .maximumSize(preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY) + .build(); + } + this.preparedStatementHandleCache.put(sql, handle); + } } // Helper class for security manager functions used by SQLServerConnection class. diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 939f4a130..f42a9adf2 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -60,7 +60,7 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** The prepared type definitions */ private String preparedTypeDefinitions; - /** The users SQL statement text */ + /** Processed SQL statement text that will be executed, may not be same as what user initially passed (which is available in sqlCommand) */ final String userSQL; /** SQL statement with expanded parameter tokens */ @@ -69,6 +69,8 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** True if this execute has been called for this statement at least once */ private boolean isExecutedAtLeastOnce = false; + private SQLServerConnection.PreparedStatementHandle cachedPreparedStatementHandle; + /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set * returned by sp_describe_parameter_encryption doesn't depend on order of input parameter @@ -91,7 +93,21 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS ArrayList batchParamValues; /** The prepared statement handle returned by the server */ - private int prepStmtHandle = 0; + private int _prepStmtHandle = 0; + private int getPrepStmtHandle() { + return _prepStmtHandle; + } + + private void setPrepStmtHandle(int handle, String sql, boolean cache) { + _prepStmtHandle = handle; + + if(cache) + this.connection.cachePreparedStatementHandle(sql, handle, executedSqlDirectly); + } + + private void resetPrepStmtHandle() { + _prepStmtHandle = 0; + } /** Flag set to true when statement execution is expected to return the prepared statement handle */ private boolean expectPrepStmtHandle = false; @@ -104,13 +120,13 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** * Used to keep track of an individual handle ready for un-prepare. */ - private final class PreparedStatementMetadataSQLCacheItem { + private final class ParsedSQLCacheItem { String preparedSQLText; int parameterCount; String procedureName; boolean bReturnValueSyntax; - PreparedStatementMetadataSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + ParsedSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { this.preparedSQLText = preparedSQLText; this.parameterCount = parameterCount; this.procedureName = procedureName; @@ -119,25 +135,25 @@ private final class PreparedStatementMetadataSQLCacheItem { } /** Size of the prepared statement meta data cache */ - static final private int preparedStatementMetadataSQLCacheSize = 100; + static final public int parsedSQLCacheSize = 100; /** Cache of prepared statement meta data */ - static private Cache preparedStatementSQLMetadataCache; + static private Cache parsedSQLCache; static { - preparedStatementSQLMetadataCache = CacheBuilder.newBuilder() - .maximumSize(preparedStatementMetadataSQLCacheSize) + parsedSQLCache = CacheBuilder.newBuilder() + .maximumSize(parsedSQLCacheSize) .build(); } /** Get prepared statement cache entry if exists */ - private PreparedStatementMetadataSQLCacheItem getCachedPreparedStatementSQLMetadata(String initialSql) { - return preparedStatementSQLMetadataCache.getIfPresent(initialSql); + private ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { + return parsedSQLCache.getIfPresent(initialSql); } /** Add cache entry for prepared statement metadata*/ - private void cachePreparedStatementSQLMetaData(String initialSql, PreparedStatementMetadataSQLCacheItem newItem) { + private void cacheParsedSQLMetadata(String initialSql, ParsedSQLCacheItem newItem) { - preparedStatementSQLMetadataCache.put(initialSql, newItem); + parsedSQLCache.put(initialSql, newItem); } // Internal function used in tracing @@ -174,7 +190,7 @@ String getClassNameInternal() { sqlCommand = sql; // Check for cached SQL metadata. - PreparedStatementMetadataSQLCacheItem cacheItem = getCachedPreparedStatementSQLMetadata(sql); + ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(sql); // No cached meta data found, parse. if(null == cacheItem) { @@ -188,8 +204,8 @@ String getClassNameInternal() { initParams(userSQL); // Cache this entry. - cacheItem = new PreparedStatementMetadataSQLCacheItem(userSQL, inOutParam.length, procedureName, bReturnValueSyntax); - cachePreparedStatementSQLMetaData(sqlCommand/*original command as key*/, cacheItem); + cacheItem = new ParsedSQLCacheItem(userSQL, inOutParam.length, procedureName, bReturnValueSyntax); + cacheParsedSQLMetadata(sqlCommand/*original command as key*/, cacheItem); } else { // Retrieve from cache item. @@ -204,7 +220,7 @@ String getClassNameInternal() { * Close the prepared statement's prepared handle. */ private void closePreparedHandle() { - if (0 == prepStmtHandle) + if (0 == getPrepStmtHandle()) return; // If the connection is already closed, don't bother trying to close @@ -212,18 +228,23 @@ private void closePreparedHandle() { // on the server anyway. if (connection.isSessionUnAvailable()) { if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().finer(this + ": Not closing PreparedHandle:" + prepStmtHandle + "; connection is already closed."); + getStatementLogger().finer(this + ": Not closing PreparedHandle:" + getPrepStmtHandle() + "; connection is already closed."); } else { isExecutedAtLeastOnce = false; - final int handleToClose = prepStmtHandle; - prepStmtHandle = 0; + final int handleToClose = getPrepStmtHandle(); + resetPrepStmtHandle(); // Using batched clean-up? If not, use old method of calling sp_unprepare. if(1 < connection.getServerPreparedStatementDiscardThreshold()) { // Handle unprepare actions through batching @ connection level. - connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); - connection.handlePreparedStatementDiscardActions(false); + // Use this only if statement caching is off, otherwise this will be called by statement cache invalidation. + if(1 > SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY) { + connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); + connection.handlePreparedStatementDiscardActions(false); + } + else if(null != cachedPreparedStatementHandle) + cachedPreparedStatementHandle.decrementRefCount(); } else { // Non batched behavior (same as pre batch impl.) @@ -567,7 +588,8 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { expectPrepStmtHandle = false; Parameter param = new Parameter(Util.shouldHonorAEForParameters(stmtColumnEncriptionSetting, connection)); param.skipRetValStatus(tdsReader); - prepStmtHandle = param.getInt(tdsReader); + int prepStmtHandle = param.getInt(tdsReader); + setPrepStmtHandle(prepStmtHandle, userSQL, true); param.skipValue(tdsReader, true); if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) getStatementLogger().finer(toString() + ": Setting PreparedHandle:" + prepStmtHandle); @@ -603,7 +625,7 @@ void sendParamsByRPC(TDSWriter tdsWriter, private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_cursorprepexec: PreparedHandle:" + prepStmtHandle + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_cursorprepexec: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = true; executedSqlDirectly = false; @@ -618,8 +640,8 @@ private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServ // // IN (reprepare): Old handle to unprepare before repreparing // OUT: The newly prepared handle - tdsWriter.writeRPCInt(null, new Integer(prepStmtHandle), true); - prepStmtHandle = 0; + tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), true); + resetPrepStmtHandle(); // OUT tdsWriter.writeRPCInt(null, new Integer(0), true); // cursor ID (OUTPUT) @@ -645,7 +667,7 @@ private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServ private void buildPrepExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_prepexec: PreparedHandle:" + prepStmtHandle + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_prepexec: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = true; executedSqlDirectly = true; @@ -660,8 +682,8 @@ private void buildPrepExecParams(TDSWriter tdsWriter) throws SQLServerException // // IN (reprepare): Old handle to unprepare before repreparing // OUT: The newly prepared handle - tdsWriter.writeRPCInt(null, new Integer(prepStmtHandle), true); - prepStmtHandle = 0; + tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), true); + resetPrepStmtHandle(); // IN tdsWriter.writeRPCStringUnicode((preparedTypeDefinitions.length() > 0) ? preparedTypeDefinitions : null); @@ -685,7 +707,7 @@ private void buildExecSQLParams(TDSWriter tdsWriter) throws SQLServerException { tdsWriter.writeByte((byte) 0); // RPC procedure option 2 // No handle used. - prepStmtHandle = 0; + resetPrepStmtHandle(); // IN tdsWriter.writeRPCStringUnicode(preparedSQL); @@ -696,7 +718,7 @@ private void buildExecSQLParams(TDSWriter tdsWriter) throws SQLServerException { private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_cursorexecute: PreparedHandle:" + prepStmtHandle + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_cursorexecute: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = false; executedSqlDirectly = false; @@ -709,8 +731,8 @@ private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerEx tdsWriter.writeByte((byte) 0); // RPC procedure option 2 */ // IN - assert 0 != prepStmtHandle; - tdsWriter.writeRPCInt(null, new Integer(prepStmtHandle), false); + assert 0 != getPrepStmtHandle(); + tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), false); // OUT tdsWriter.writeRPCInt(null, new Integer(0), true); @@ -727,7 +749,7 @@ private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerEx private void buildExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_execute: PreparedHandle:" + prepStmtHandle + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_execute: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = false; executedSqlDirectly = true; @@ -740,8 +762,8 @@ private void buildExecParams(TDSWriter tdsWriter) throws SQLServerException { tdsWriter.writeByte((byte) 0); // RPC procedure option 2 */ // IN - assert 0 != prepStmtHandle; - tdsWriter.writeRPCInt(null, new Integer(prepStmtHandle), false); + assert 0 != getPrepStmtHandle(); + tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), false); } private void getParameterEncryptionMetadata(Parameter[] params) throws SQLServerException { @@ -889,7 +911,7 @@ private boolean doPrepExec(TDSWriter tdsWriter, Parameter[] params, boolean hasNewTypeDefinitions) throws SQLServerException { - boolean needsPrepare = hasNewTypeDefinitions || 0 == prepStmtHandle; + boolean needsPrepare = hasNewTypeDefinitions || 0 == getPrepStmtHandle(); // Cursors never go the non-prepared statement route. if (isCursorable(executeMethod)) { @@ -899,17 +921,31 @@ private boolean doPrepExec(TDSWriter tdsWriter, buildServerCursorExecParams(tdsWriter); } else { - // Move overhead of needing to do prepare & unprepare to only use cases that need more than one execution. - // First execution, use sp_executesql, optimizing for asumption we will not re-use statement. - if (!connection.getEnablePrepareOnFirstPreparedStatementCall() && !isExecutedAtLeastOnce) { - buildExecSQLParams(tdsWriter); - isExecutedAtLeastOnce = true; - } - // Second execution, use prepared statements since we seem to be re-using it. - else if(needsPrepare) - buildPrepExecParams(tdsWriter); - else + // Check for cached handle. + SQLServerConnection.PreparedStatementHandle cachedHandle = this.connection.getCachedPreparedStatementHandle(userSQL); + + // If handle was found then re-use. + if(null != cachedHandle && cachedHandle.incrementRefCountAndVerifyNotInvalidated()) { + cachedPreparedStatementHandle = cachedHandle; + + setPrepStmtHandle(cachedHandle.handle, userSQL, false); + needsPrepare = false; + buildExecParams(tdsWriter); + } + else { + // Move overhead of needing to do prepare & unprepare to only use cases that need more than one execution. + // First execution, use sp_executesql, optimizing for asumption we will not re-use statement. + if (!connection.getEnablePrepareOnFirstPreparedStatementCall() && !isExecutedAtLeastOnce) { + buildExecSQLParams(tdsWriter); + isExecutedAtLeastOnce = true; + } + // Second execution, use prepared statements since we seem to be re-using it. + else if(needsPrepare) + buildPrepExecParams(tdsWriter); + else + buildExecParams(tdsWriter); + } } sendParamsByRPC(tdsWriter, params); diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index e6b94bf7c..03ed1167c 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -55,6 +55,9 @@ private int executeSQLReturnFirstInt(SQLServerConnection conn, String sql) throw public void testBatchedUnprepare() throws SQLException { SQLServerConnection conOuter = null; + // Turn off use of prepared statement cache. + SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 0; + // Make sure correct settings are used. SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); @@ -137,6 +140,9 @@ public void testBatchedUnprepare() throws SQLException { @Test public void testPreparedStatementExecAndUnprepareConfig() throws SQLException { + // Turn off use of prepared statement cache. + SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 0; + // Verify initial defaults are correct: assertTrue(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold() > 1); assertTrue(false == SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); From 1856ef5480092d636cd74cbd402d21b75efc3da9 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Sun, 30 Apr 2017 09:17:41 -0500 Subject: [PATCH 06/34] Removed unnec. parsing per Brett's comment --- .../microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index f42a9adf2..18c072119 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -988,7 +988,7 @@ else if (resultSet != null) { /* L0 */ private ResultSet buildExecuteMetaData() throws SQLServerException { String fmtSQL = sqlCommand; if (fmtSQL.indexOf(LEFT_CURLY_BRACKET) >= 0) { - fmtSQL = (new JDBCSyntaxTranslator()).translate(fmtSQL); + fmtSQL = userSQL; } ResultSet emptyResultSet = null; From 503f5394abc06db927dc444434368ad174deb560 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Wed, 3 May 2017 22:45:32 -0500 Subject: [PATCH 07/34] Updated prototype for statement and parameter metadata pooling. --- .../sqlserver/jdbc/SQLServerConnection.java | 182 +++++++++--- .../sqlserver/jdbc/SQLServerDataSource.java | 20 ++ .../sqlserver/jdbc/SQLServerDriver.java | 33 ++- .../jdbc/SQLServerParsedSQLCacheItem.java | 27 ++ .../jdbc/SQLServerPreparedStatement.java | 258 +++++++++++------- .../sqlserver/jdbc/SQLServerResource.java | 2 + .../sqlserver/jdbc/SQLServerStatement.java | 15 +- .../unit/statement/PreparedStatementTest.java | 85 +++++- 8 files changed, 454 insertions(+), 168 deletions(-) create mode 100644 src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 0d4a3c96e..270c16f74 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -123,30 +123,44 @@ public class SQLServerConnection implements ISQLServerConnection { /** * Used to keep track of an individual handle ready for un-prepare. */ - final class PreparedStatementHandle { + final class PreparedStatementCacheItem { int handle; - boolean directSql; + boolean hasExecutedSpExecuteSql; + boolean handleIsDirectSql; SQLServerConnection connection; private AtomicInteger refCount = new AtomicInteger(1); + SQLServerParameterMetaData parameterMetadata; - PreparedStatementHandle(int handle, boolean directSql, SQLServerConnection connection) { + PreparedStatementCacheItem(int handle, boolean handleIsDirectSql, boolean hasExecutedSpExecuteSql, SQLServerParameterMetaData parameterMetadata, SQLServerConnection connection) { this.handle = handle; - this.directSql = directSql; + this.handleIsDirectSql = handleIsDirectSql; + this.hasExecutedSpExecuteSql = hasExecutedSpExecuteSql; this.connection = connection; + this.parameterMetadata = parameterMetadata; + } + + boolean hasHandle() { + return 0 < this.handle; + } + + boolean hasParameterMetadata() { + return null != this.parameterMetadata; } // Returns false if handle is not re-usable. - boolean incrementRefCountAndVerifyNotInvalidated() { + boolean incrementHandleRefCountAndVerifyNotInvalidated(SQLServerPreparedStatement statement) { // If refcount is negative the handle has been killed. if(0 > this.refCount.getAndIncrement()) { this.refCount.getAndDecrement(); // Reduce again. return false; } - else - return true; + else { + statement.cachedPreparedStatementHandle = this; + return true; + } } - - boolean discardIfNotReferenced() { + + boolean discardIfHandleNotReferenced() { // If refcount is zero or negative the handle can be killed. if(1 > this.refCount.getAndDecrement()) { return true; @@ -158,16 +172,16 @@ boolean discardIfNotReferenced() { } } - void decrementRefCount() { + void decrementHandleRefCount() { this.refCount.decrementAndGet(); } } - /** Size of the prepared statement meta data cache */ - static public int preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 10; + /** Size of the prepared statement handle cache */ + private int statementPoolingCacheSize = 10; - /** Cache of prepared statement meta data */ - private Cache preparedStatementHandleCache; + /** Cache of prepared statement handles */ + private Cache preparedStatementCache; SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; @@ -1246,14 +1260,28 @@ Connection connectInternal(Properties propsIn, sendTimeAsDatetime = booleanPropertyOn(sPropKey, sPropValue); - sPropKey = SQLServerDriverBooleanProperty.DISABLE_STATEMENT_POOLING.toString(); - sPropValue = activeConnectionProperties.getProperty(sPropKey); - if (sPropValue != null) // if the user does not set it, it is ok but if set the value can only be true - if (false == booleanPropertyOn(sPropKey, sPropValue)) { - MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_invaliddisableStatementPooling")); - Object[] msgArgs = {new String(sPropValue)}; + // Must be set before DISABLE_STATEMENT_POOLING + sPropKey = SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.toString(); + if (activeConnectionProperties.getProperty(sPropKey) != null && activeConnectionProperties.getProperty(sPropKey).length() > 0) { + try { + int n = (new Integer(activeConnectionProperties.getProperty(sPropKey))).intValue(); + this.setStatementPoolingCacheSize(n); + } + catch (NumberFormatException e) { + MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_statementPoolingCacheSize")); + Object[] msgArgs = {activeConnectionProperties.getProperty(sPropKey)}; SQLServerException.makeFromDriverError(this, this, form.format(msgArgs), null, false); } + } + + // Must be set after STATEMENT_POOLING_CACHE_SIZE + sPropKey = SQLServerDriverBooleanProperty.DISABLE_STATEMENT_POOLING.toString(); + sPropValue = activeConnectionProperties.getProperty(sPropKey); + if (null != sPropValue) { + // If disabled set cache size to 0 if disabled. + if(booleanPropertyOn(sPropKey, sPropValue)) + this.setStatementPoolingCacheSize(0); + } sPropKey = SQLServerDriverBooleanProperty.INTEGRATED_SECURITY.toString(); sPropValue = activeConnectionProperties.getProperty(sPropKey); @@ -2726,6 +2754,10 @@ public void close() throws SQLServerException { tdsChannel.close(); } + // Invalidate statement cache. + if(null != this.preparedStatementCache) + this.preparedStatementCache.invalidateAll(); + // Clean-up queue etc. related to batching of prepared statement discard actions (sp_unprepare). cleanupPreparedStatementDiscardActions(); @@ -5492,47 +5524,119 @@ final void handlePreparedStatementDiscardActions(boolean force) { } + /** + * Returns the size of the prepared statement cache for this connection. A value less than 1 means no cache. + * @return Returns the current setting per the description. + */ + public int getStatementPoolingCacheSize() { + return this.statementPoolingCacheSize; + } + + /** + * Whether statement pooling is enabled or not for this connection. + * @return Returns the current setting per the description. + */ + public boolean isStatementPoolingEnabled() { + return 0 < this.getStatementPoolingCacheSize(); + } + + /** + * Specifies the size of the prepared statement cache for this conection. A value less than 1 means no cache. + * @value The new cache size. + */ + public void setStatementPoolingCacheSize(int value) { + this.statementPoolingCacheSize = value; + } + /** Get prepared statement cache entry if exists */ - final PreparedStatementHandle getCachedPreparedStatementHandle(String sql) { - if(null == this.preparedStatementHandleCache) + final PreparedStatementCacheItem getCachedPreparedStatementMetadata(String sql) { + if(null == this.preparedStatementCache) return null; - return this.preparedStatementHandleCache.getIfPresent(sql); + PreparedStatementCacheItem cacheItem = this.preparedStatementCache.getIfPresent(sql); + + return cacheItem; } // Handle closing handles when removed from cache. - RemovalListener preparedStatementHandleCacheRemovalListener = new RemovalListener() { - public void onRemoval(RemovalNotification removal) { - PreparedStatementHandle handle = removal.getValue(); + RemovalListener preparedStatementHandleCacheRemovalListener = new RemovalListener() { + public void onRemoval(RemovalNotification removal) { + PreparedStatementCacheItem cacheItem = removal.getValue(); // Only discard if not referenced. - if(null != handle && handle.discardIfNotReferenced()) { - handle.connection.enqueuePreparedStatementDiscardItem(handle.handle, handle.directSql); - handle.connection.handlePreparedStatementDiscardActions(false); + if(null != cacheItem && cacheItem.discardIfHandleNotReferenced()) { + if(cacheItem.hasHandle()) { + cacheItem.connection.enqueuePreparedStatementDiscardItem(cacheItem.handle, cacheItem.handleIsDirectSql); + cacheItem.connection.handlePreparedStatementDiscardActions(false); + } } - else if(null != handle) + else if(null != cacheItem) // Put back in cache. - handle.connection.cachePreparedStatementHandle(removal.getKey(), handle); + cacheItem.connection.cachePreparedStatementMetadata(removal.getKey(), cacheItem); } }; /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementHandle(String sql, int handle, boolean directSql) { - this.cachePreparedStatementHandle(sql, new PreparedStatementHandle(handle, directSql, this)); + final void cachePreparedStatementExecuteSqlUse(String sql) { + // Caching turned off? + if(0 >= this.getStatementPoolingCacheSize()) + return; + + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + + if(null != cacheItem) + cacheItem.hasExecutedSpExecuteSql = true; + else { + cacheItem = new PreparedStatementCacheItem(0, false, true, null, this); + + this.cachePreparedStatementMetadata(sql, cacheItem); + } + } + + /** Add cache entry for prepared statement metadata*/ + final void cachePreparedStatementHandle(String sql, int handle, boolean directSql, SQLServerPreparedStatement statement) { + // Caching turned off? + if(0 >= this.getStatementPoolingCacheSize()) + return; + + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + + if(null != cacheItem) { + cacheItem.handle = handle; + cacheItem.handleIsDirectSql = directSql; + } + else { + cacheItem = new PreparedStatementCacheItem(handle, directSql, false, null, this); + + this.cachePreparedStatementMetadata(sql, cacheItem); + } + } + + /** Add cache entry for prepared statement metadata*/ + final void cacheParameterMetadata(String sql, SQLServerParameterMetaData metadata, SQLServerPreparedStatement statement) { + // Caching turned off? + if(0 >= this.getStatementPoolingCacheSize()) + return; + + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + if(null != cacheItem) + cacheItem.parameterMetadata = metadata; + else + this.cachePreparedStatementMetadata(sql, new PreparedStatementCacheItem(0, false, false, metadata, this)); } /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementHandle(String sql, PreparedStatementHandle handle) { + private void cachePreparedStatementMetadata(String sql, PreparedStatementCacheItem cacheItem) { // Caching turned off? - if(0 >= SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY || null == handle) + if(0 >= this.getStatementPoolingCacheSize() || null == cacheItem) return; - if(null == this.preparedStatementHandleCache) { - preparedStatementHandleCache = CacheBuilder.newBuilder() - .maximumSize(preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY) + if(null == this.preparedStatementCache) { + preparedStatementCache = CacheBuilder.newBuilder() + .maximumSize(this.getStatementPoolingCacheSize()) .build(); } - this.preparedStatementHandleCache.put(sql, handle); + this.preparedStatementCache.put(sql, cacheItem); } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java index 54a65a28d..cd9d4ab59 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java @@ -713,6 +713,26 @@ public int getServerPreparedStatementDiscardThreshold() { SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); } + /** + * Specifies the size of the prepared statement cache for this conection. A value less than 1 means no cache. + * + * @param statementPoolingCacheSize + * Changes the setting per the description. + */ + public void setStatementPoolingCacheSize(int statementPoolingCacheSize) { + setIntProperty(connectionProps, SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.toString(), statementPoolingCacheSize); + } + + /** + * Returns the size of the prepared statement cache for this conection. A value less than 1 means no cache. + * + * @return Returns the current setting per the description. + */ + public int getStatementPoolingCacheSize() { + int defaultSize = SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.getDefaultValue(); + return getIntProperty(connectionProps, SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.toString(), defaultSize); + } + public void setSocketTimeout(int socketTimeout) { setIntProperty(connectionProps, SQLServerDriverIntProperty.SOCKET_TIMEOUT.toString(), socketTimeout); } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java index 9b80a033b..d6a7049aa 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java @@ -265,13 +265,15 @@ public String toString() { } enum SQLServerDriverIntProperty { - PACKET_SIZE ("packetSize", TDS.DEFAULT_PACKET_SIZE), - LOCK_TIMEOUT ("lockTimeout", -1), - LOGIN_TIMEOUT ("loginTimeout", 15), - QUERY_TIMEOUT ("queryTimeout", -1), - PORT_NUMBER ("portNumber", 1433), - SOCKET_TIMEOUT ("socketTimeout", 0), - SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD("serverPreparedStatementDiscardThreshold", -1/*This is not the default, default handled in SQLServerConnection and is not final/const*/); + PACKET_SIZE ("packetSize", TDS.DEFAULT_PACKET_SIZE), + LOCK_TIMEOUT ("lockTimeout", -1), + LOGIN_TIMEOUT ("loginTimeout", 15), + QUERY_TIMEOUT ("queryTimeout", -1), + PORT_NUMBER ("portNumber", 1433), + SOCKET_TIMEOUT ("socketTimeout", 0), + SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD("serverPreparedStatementDiscardThreshold", -1/*This is not the default, default handled in SQLServerConnection and is not final/const*/), + STATEMENT_POOLING_CACHE_SIZE ("statementPoolingCacheSize", 10), + ; private String name; private int defaultValue; @@ -291,9 +293,9 @@ public String toString() { } } -enum SQLServerDriverBooleanProperty +enum SQLServerDriverBooleanProperty { - DISABLE_STATEMENT_POOLING ("disableStatementPooling", true), + DISABLE_STATEMENT_POOLING ("disableStatementPooling", false), ENCRYPT ("encrypt", false), INTEGRATED_SECURITY ("integratedSecurity", false), LAST_UPDATE_COUNT ("lastUpdateCount", true), @@ -334,10 +336,10 @@ public final class SQLServerDriver implements java.sql.Driver { { // default required available choices // property name value property (if appropriate) - new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.APPLICATION_INTENT.toString(), SQLServerDriverStringProperty.APPLICATION_INTENT.getDefaultValue(), false, new String[]{ApplicationIntent.READ_ONLY.toString(), ApplicationIntent.READ_WRITE.toString()}), - new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.APPLICATION_NAME.toString(), SQLServerDriverStringProperty.APPLICATION_NAME.getDefaultValue(), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.APPLICATION_INTENT.toString(), SQLServerDriverStringProperty.APPLICATION_INTENT.getDefaultValue(), false, new String[]{ApplicationIntent.READ_ONLY.toString(), ApplicationIntent.READ_WRITE.toString()}), + new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.APPLICATION_NAME.toString(), SQLServerDriverStringProperty.APPLICATION_NAME.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.COLUMN_ENCRYPTION.toString(), SQLServerDriverStringProperty.COLUMN_ENCRYPTION.getDefaultValue(), false, new String[] {ColumnEncryptionSetting.Disabled.toString(), ColumnEncryptionSetting.Enabled.toString()}), - new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.DATABASE_NAME.toString(), SQLServerDriverStringProperty.DATABASE_NAME.getDefaultValue(), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.DATABASE_NAME.toString(), SQLServerDriverStringProperty.DATABASE_NAME.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.DISABLE_STATEMENT_POOLING.toString(), Boolean.toString(SQLServerDriverBooleanProperty.DISABLE_STATEMENT_POOLING.getDefaultValue()), false, new String[] {"true"}), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.ENCRYPT.toString(), Boolean.toString(SQLServerDriverBooleanProperty.ENCRYPT.getDefaultValue()), false, TRUE_FALSE), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.FAILOVER_PARTNER.toString(), SQLServerDriverStringProperty.FAILOVER_PARTNER.getDefaultValue(), false, null), @@ -351,7 +353,7 @@ public final class SQLServerDriver implements java.sql.Driver { new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.LOCK_TIMEOUT.toString(), Integer.toString(SQLServerDriverIntProperty.LOCK_TIMEOUT.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.LOGIN_TIMEOUT.toString(), Integer.toString(SQLServerDriverIntProperty.LOGIN_TIMEOUT.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.MULTI_SUBNET_FAILOVER.toString(), Boolean.toString(SQLServerDriverBooleanProperty.MULTI_SUBNET_FAILOVER.getDefaultValue()), false, TRUE_FALSE), - new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.PACKET_SIZE.toString(), Integer.toString(SQLServerDriverIntProperty.PACKET_SIZE.getDefaultValue()), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.PACKET_SIZE.toString(), Integer.toString(SQLServerDriverIntProperty.PACKET_SIZE.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.PASSWORD.toString(), SQLServerDriverStringProperty.PASSWORD.getDefaultValue(), true, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.PORT_NUMBER.toString(), Integer.toString(SQLServerDriverIntProperty.PORT_NUMBER.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.QUERY_TIMEOUT.toString(), Integer.toString(SQLServerDriverIntProperty.QUERY_TIMEOUT.getDefaultValue()), false, null), @@ -368,15 +370,16 @@ public final class SQLServerDriver implements java.sql.Driver { new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(), SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.toString(), Boolean.toString(SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.getDefaultValue()), false, TRUE_FALSE), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.USER.toString(), SQLServerDriverStringProperty.USER.getDefaultValue(), true, null), - new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.WORKSTATION_ID.toString(), SQLServerDriverStringProperty.WORKSTATION_ID.getDefaultValue(), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.WORKSTATION_ID.toString(), SQLServerDriverStringProperty.WORKSTATION_ID.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.XOPEN_STATES.toString(), Boolean.toString(SQLServerDriverBooleanProperty.XOPEN_STATES.getDefaultValue()), false, TRUE_FALSE), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.AUTHENTICATION_SCHEME.toString(), SQLServerDriverStringProperty.AUTHENTICATION_SCHEME.getDefaultValue(), false, new String[] {AuthenticationScheme.javaKerberos.toString(),AuthenticationScheme.nativeAuthentication.toString()}), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.AUTHENTICATION.toString(), SQLServerDriverStringProperty.AUTHENTICATION.getDefaultValue(), false, new String[] {SqlAuthentication.NotSpecified.toString(),SqlAuthentication.SqlPassword.toString(),SqlAuthentication.ActiveDirectoryPassword.toString(),SqlAuthentication.ActiveDirectoryIntegrated.toString()}), - new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.FIPS_PROVIDER.toString(), SQLServerDriverStringProperty.FIPS_PROVIDER.getDefaultValue(), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.FIPS_PROVIDER.toString(), SQLServerDriverStringProperty.FIPS_PROVIDER.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.SOCKET_TIMEOUT.toString(), Integer.toString(SQLServerDriverIntProperty.SOCKET_TIMEOUT.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.FIPS.toString(), Boolean.toString(SQLServerDriverBooleanProperty.FIPS.getDefaultValue()), false, TRUE_FALSE), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), Boolean.toString(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()), false, TRUE_FALSE), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.toString(), Integer.toString(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.toString(), Integer.toString(SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.getDefaultValue()), false, null), }; // Properties that can only be set by using Properties. diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java new file mode 100644 index 000000000..0ae2db889 --- /dev/null +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java @@ -0,0 +1,27 @@ +/* + * Microsoft JDBC Driver for SQL Server + * + * Copyright(c) Microsoft Corporation All rights reserved. + * + * This program is made available under the terms of the MIT License. See the LICENSE file in the project root for more information. + */ + +package com.microsoft.sqlserver.jdbc; + +/** + * Used to keep track of parsed SQL text and its properties for prepared statements. + */ +final class ParsedSQLCacheItem { + /** The SQL text AFTER processing. */ + String preparedSQLText; + int parameterCount; + String procedureName; + boolean bReturnValueSyntax; + + ParsedSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + this.preparedSQLText = preparedSQLText; + this.parameterCount = parameterCount; + this.procedureName = procedureName; + this.bReturnValueSyntax = bReturnValueSyntax; + } +} diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 18c072119..9dc8b95cf 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -69,7 +69,8 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** True if this execute has been called for this statement at least once */ private boolean isExecutedAtLeastOnce = false; - private SQLServerConnection.PreparedStatementHandle cachedPreparedStatementHandle; + /** Reference to cache item for statement pooling. Only used to decrement ref count on statement close. */ + SQLServerConnection.PreparedStatementCacheItem cachedPreparedStatementHandle; /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set @@ -93,20 +94,45 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS ArrayList batchParamValues; /** The prepared statement handle returned by the server */ - private int _prepStmtHandle = 0; - private int getPrepStmtHandle() { - return _prepStmtHandle; + private int prepStmtHandle = 0; + + /** The server handle for this prepared statement. If a value < 1 is returned no handle has been created. + * + * @return + * Per the description. + */ + public int getPreparedStatementHandle() { + return prepStmtHandle; + } + + /** Returns true if this statement has a server handle. + * + * @return + * Per the description. + */ + private boolean hasPreparedStatementHandle() { + return 0 < prepStmtHandle; } + /** Sets the server handle for this prepared statement. + * + * @handle + * @sql + * @cache + */ private void setPrepStmtHandle(int handle, String sql, boolean cache) { - _prepStmtHandle = handle; + assert 0 < handle; + + prepStmtHandle = handle; - if(cache) - this.connection.cachePreparedStatementHandle(sql, handle, executedSqlDirectly); + if(cache) + connection.cachePreparedStatementHandle(sql, handle, executedSqlDirectly, this); } + /** Resets the server handle for this prepared statement to no handle. + */ private void resetPrepStmtHandle() { - _prepStmtHandle = 0; + prepStmtHandle = 0; } /** Flag set to true when statement execution is expected to return the prepared statement handle */ @@ -117,25 +143,9 @@ private void resetPrepStmtHandle() { */ private boolean encryptionMetadataIsRetrieved = false; - /** - * Used to keep track of an individual handle ready for un-prepare. - */ - private final class ParsedSQLCacheItem { - String preparedSQLText; - int parameterCount; - String procedureName; - boolean bReturnValueSyntax; - - ParsedSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { - this.preparedSQLText = preparedSQLText; - this.parameterCount = parameterCount; - this.procedureName = procedureName; - this.bReturnValueSyntax = bReturnValueSyntax; - } - } - /** Size of the prepared statement meta data cache */ - static final public int parsedSQLCacheSize = 100; + /** Size of the parsed SQL-text metadata cache */ + static final private int parsedSQLCacheSize = 100; /** Cache of prepared statement meta data */ static private Cache parsedSQLCache; @@ -146,14 +156,25 @@ private final class ParsedSQLCacheItem { } /** Get prepared statement cache entry if exists */ - private ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { + static ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { return parsedSQLCache.getIfPresent(initialSql); } /** Add cache entry for prepared statement metadata*/ - private void cacheParsedSQLMetadata(String initialSql, ParsedSQLCacheItem newItem) { - - parsedSQLCache.put(initialSql, newItem); + static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql) throws SQLServerException { + + JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); + + String parsedSql = translator.translate(initialSql); + String procName = translator.getProcedureName(); // may return null + boolean returnValueSyntax = translator.hasReturnValueSyntax(); + int paramCount = countParams(parsedSql); + + // Cache this entry. + ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); + parsedSQLCache.put(initialSql, cacheItem); + + return cacheItem; } // Internal function used in tracing @@ -193,34 +214,24 @@ String getClassNameInternal() { ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(sql); // No cached meta data found, parse. - if(null == cacheItem) { - JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); - - userSQL = translator.translate(sql); - procedureName = translator.getProcedureName(); // may return null - bReturnValueSyntax = translator.hasReturnValueSyntax(); + if(null == cacheItem) + cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql); - // Save processed SQL statement. - initParams(userSQL); + // Retrieve from cache item. + procedureName = cacheItem.procedureName; + bReturnValueSyntax = cacheItem.bReturnValueSyntax; + userSQL = cacheItem.preparedSQLText; + initParams(cacheItem.parameterCount); - // Cache this entry. - cacheItem = new ParsedSQLCacheItem(userSQL, inOutParam.length, procedureName, bReturnValueSyntax); - cacheParsedSQLMetadata(sqlCommand/*original command as key*/, cacheItem); - } - else { - // Retrieve from cache item. - procedureName = cacheItem.procedureName; - bReturnValueSyntax = cacheItem.bReturnValueSyntax; - userSQL = cacheItem.preparedSQLText; - initParams(cacheItem.parameterCount); - } + // See if existing handle can be re-used. + handleUsingCachedStmtHandle(); } /** * Close the prepared statement's prepared handle. */ private void closePreparedHandle() { - if (0 == getPrepStmtHandle()) + if (!hasPreparedStatementHandle()) return; // If the connection is already closed, don't bother trying to close @@ -228,23 +239,23 @@ private void closePreparedHandle() { // on the server anyway. if (connection.isSessionUnAvailable()) { if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().finer(this + ": Not closing PreparedHandle:" + getPrepStmtHandle() + "; connection is already closed."); + getStatementLogger().finer(this + ": Not closing PreparedHandle:" + getPreparedStatementHandle() + "; connection is already closed."); } else { isExecutedAtLeastOnce = false; - final int handleToClose = getPrepStmtHandle(); + final int handleToClose = getPreparedStatementHandle(); resetPrepStmtHandle(); // Using batched clean-up? If not, use old method of calling sp_unprepare. if(1 < connection.getServerPreparedStatementDiscardThreshold()) { // Handle unprepare actions through batching @ connection level. // Use this only if statement caching is off, otherwise this will be called by statement cache invalidation. - if(1 > SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY) { + if(!this.connection.isStatementPoolingEnabled()) { connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); connection.handlePreparedStatementDiscardActions(false); } - else if(null != cachedPreparedStatementHandle) - cachedPreparedStatementHandle.decrementRefCount(); + else if(null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) + cachedPreparedStatementHandle.decrementHandleRefCount(); } else { // Non batched behavior (same as pre batch impl.) @@ -301,12 +312,12 @@ final void closeInternal() { } /** - * Find and intialize the statement parameters. + * Find statement parameters. * * @param sql * SQL text to parse for number of parameters to intialize. */ - /* L0 */ final void initParams(String sql) { + static int countParams(String sql) { int nParams = 0; // Figure out the expected number of parameters by counting the @@ -315,10 +326,10 @@ final void closeInternal() { while ((offset = ParameterUtils.scanSQLForChar('?', sql, ++offset)) < sql.length()) ++nParams; - initParams(nParams); + return nParams; } - /** + /** * Intialize the statement parameters. * * @param nParams @@ -529,6 +540,7 @@ final void doExecutePreparedStatement(PrepStmtExecCmd command) throws SQLServerE loggerExternal.finer(toString() + " ActivityId: " + ActivityCorrelator.getNext().toString()); } + boolean hasExistingTypeDefinitions = preparedTypeDefinitions != null; boolean hasNewTypeDefinitions = true; if (!encryptionMetadataIsRetrieved) { hasNewTypeDefinitions = buildPreparedStrings(inOutParam, false); @@ -554,7 +566,7 @@ final void doExecutePreparedStatement(PrepStmtExecCmd command) throws SQLServerE // continue using it after we return. TDSWriter tdsWriter = command.startRequest(TDS.PKT_RPC); - doPrepExec(tdsWriter, inOutParam, hasNewTypeDefinitions); + doPrepExec(tdsWriter, inOutParam, hasNewTypeDefinitions, hasExistingTypeDefinitions); ensureExecuteResultsReader(command.startResponse(getIsResponseBufferingAdaptive())); startResults(); @@ -568,6 +580,26 @@ else if (EXECUTE_UPDATE == executeMethod && null != resultSet) { } } + private void handleUsingCachedStmtHandle() { + if(!hasPreparedStatementHandle()) { + // Check for cached handle. + SQLServerConnection.PreparedStatementCacheItem cachedHandle = this.connection.getCachedPreparedStatementMetadata(userSQL); + + // If handle was found then re-use. + if(null != cachedHandle && (cachedHandle.hasHandle() || cachedHandle.hasExecutedSpExecuteSql)) { + + // If existing handle was found use it and specify no need for prepare. + if(cachedHandle.hasHandle() && cachedHandle.incrementHandleRefCountAndVerifyNotInvalidated(this)) { + setPrepStmtHandle(cachedHandle.handle, userSQL, false); + } + else { + // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. + isExecutedAtLeastOnce = true; + } + } + } + } + /** * Consume the OUT parameter for the statement object itself. * @@ -625,7 +657,7 @@ void sendParamsByRPC(TDSWriter tdsWriter, private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_cursorprepexec: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_cursorprepexec: PreparedHandle:" + getPreparedStatementHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = true; executedSqlDirectly = false; @@ -640,7 +672,7 @@ private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServ // // IN (reprepare): Old handle to unprepare before repreparing // OUT: The newly prepared handle - tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), true); + tdsWriter.writeRPCInt(null, new Integer(getPreparedStatementHandle()), true); resetPrepStmtHandle(); // OUT @@ -667,7 +699,7 @@ private void buildServerCursorPrepExecParams(TDSWriter tdsWriter) throws SQLServ private void buildPrepExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_prepexec: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_prepexec: PreparedHandle:" + getPreparedStatementHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = true; executedSqlDirectly = true; @@ -682,7 +714,7 @@ private void buildPrepExecParams(TDSWriter tdsWriter) throws SQLServerException // // IN (reprepare): Old handle to unprepare before repreparing // OUT: The newly prepared handle - tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), true); + tdsWriter.writeRPCInt(null, new Integer(getPreparedStatementHandle()), true); resetPrepStmtHandle(); // IN @@ -718,7 +750,7 @@ private void buildExecSQLParams(TDSWriter tdsWriter) throws SQLServerException { private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_cursorexecute: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_cursorexecute: PreparedHandle:" + getPreparedStatementHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = false; executedSqlDirectly = false; @@ -731,8 +763,8 @@ private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerEx tdsWriter.writeByte((byte) 0); // RPC procedure option 2 */ // IN - assert 0 != getPrepStmtHandle(); - tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), false); + assert hasPreparedStatementHandle(); + tdsWriter.writeRPCInt(null, new Integer(getPreparedStatementHandle()), false); // OUT tdsWriter.writeRPCInt(null, new Integer(0), true); @@ -749,7 +781,7 @@ private void buildServerCursorExecParams(TDSWriter tdsWriter) throws SQLServerEx private void buildExecParams(TDSWriter tdsWriter) throws SQLServerException { if (getStatementLogger().isLoggable(java.util.logging.Level.FINE)) - getStatementLogger().fine(toString() + ": calling sp_execute: PreparedHandle:" + getPrepStmtHandle() + ", SQL:" + preparedSQL); + getStatementLogger().fine(toString() + ": calling sp_execute: PreparedHandle:" + getPreparedStatementHandle() + ", SQL:" + preparedSQL); expectPrepStmtHandle = false; executedSqlDirectly = true; @@ -762,8 +794,8 @@ private void buildExecParams(TDSWriter tdsWriter) throws SQLServerException { tdsWriter.writeByte((byte) 0); // RPC procedure option 2 */ // IN - assert 0 != getPrepStmtHandle(); - tdsWriter.writeRPCInt(null, new Integer(getPrepStmtHandle()), false); + assert hasPreparedStatementHandle(); + tdsWriter.writeRPCInt(null, new Integer(getPreparedStatementHandle()), false); } private void getParameterEncryptionMetadata(Parameter[] params) throws SQLServerException { @@ -909,9 +941,10 @@ private void getParameterEncryptionMetadata(Parameter[] params) throws SQLServer private boolean doPrepExec(TDSWriter tdsWriter, Parameter[] params, - boolean hasNewTypeDefinitions) throws SQLServerException { + boolean hasNewTypeDefinitions, + boolean hasExistingTypeDefinitions) throws SQLServerException { - boolean needsPrepare = hasNewTypeDefinitions || 0 == getPrepStmtHandle(); + boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasPreparedStatementHandle(); // Cursors never go the non-prepared statement route. if (isCursorable(executeMethod)) { @@ -921,31 +954,20 @@ private boolean doPrepExec(TDSWriter tdsWriter, buildServerCursorExecParams(tdsWriter); } else { - // Check for cached handle. - SQLServerConnection.PreparedStatementHandle cachedHandle = this.connection.getCachedPreparedStatementHandle(userSQL); - - // If handle was found then re-use. - if(null != cachedHandle && cachedHandle.incrementRefCountAndVerifyNotInvalidated()) { - cachedPreparedStatementHandle = cachedHandle; - - setPrepStmtHandle(cachedHandle.handle, userSQL, false); - needsPrepare = false; - - buildExecParams(tdsWriter); - } - else { - // Move overhead of needing to do prepare & unprepare to only use cases that need more than one execution. - // First execution, use sp_executesql, optimizing for asumption we will not re-use statement. - if (!connection.getEnablePrepareOnFirstPreparedStatementCall() && !isExecutedAtLeastOnce) { - buildExecSQLParams(tdsWriter); - isExecutedAtLeastOnce = true; - } - // Second execution, use prepared statements since we seem to be re-using it. - else if(needsPrepare) - buildPrepExecParams(tdsWriter); - else - buildExecParams(tdsWriter); + // Move overhead of needing to do prepare & unprepare to only use cases that need more than one execution. + // First execution, use sp_executesql, optimizing for asumption we will not re-use statement. + if (needsPrepare && !connection.getEnablePrepareOnFirstPreparedStatementCall() && !isExecutedAtLeastOnce) { + buildExecSQLParams(tdsWriter); + isExecutedAtLeastOnce = true; + + // Enable re-use if caching is on by moving to sp_prepexec on next call even from separate instance. + connection.cachePreparedStatementExecuteSqlUse(userSQL); } + // Second execution, use prepared statements since we seem to be re-using it. + else if(needsPrepare) + buildPrepExecParams(tdsWriter); + else + buildExecParams(tdsWriter); } sendParamsByRPC(tdsWriter, params); @@ -2524,8 +2546,10 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th assert paramValues.length == batchParam.length; for (int i = 0; i < paramValues.length; i++) batchParam[i] = paramValues[i]; - + + boolean hasExistingTypeDefinitions = preparedTypeDefinitions != null; boolean hasNewTypeDefinitions = buildPreparedStrings(batchParam, false); + // Get the encryption metadata for the first batch only. if ((0 == numBatchesExecuted) && (Util.shouldHonorAEForParameters(stmtColumnEncriptionSetting, connection)) && (0 < batchParam.length) && !isInternalEncryptionQuery) { @@ -2566,7 +2590,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th // the size of a batch's string parameter values changes such // that repreparation is necessary. ++numBatchesPrepared; - if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions) || numBatchesPrepared == numBatches) { + if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions, hasExistingTypeDefinitions) || numBatchesPrepared == numBatches) { ensureExecuteResultsReader(batchCommand.startResponse(getIsResponseBufferingAdaptive())); while (numBatchesExecuted < numBatchesPrepared) { @@ -2892,14 +2916,42 @@ public final void setNull(int paramIndex, loggerExternal.exiting(getClassNameLogging(), "setNull"); } + /** + * Returns parameter metadata for the prepared statement. + * + * @forceRefresh + * If true the cache will not be used to retrieve the metadata. + * + * @return + * Per the description. + */ + public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws SQLServerException { + + SQLServerConnection.PreparedStatementCacheItem cacheItem = null; + if( + !forceRefresh + && null != (cacheItem = connection.getCachedPreparedStatementMetadata(userSQL)) + && cacheItem.hasParameterMetadata() + ) { + return cacheItem.parameterMetadata; + } + else { + loggerExternal.entering(getClassNameLogging(), "getParameterMetaData"); + checkClosed(); + SQLServerParameterMetaData pmd = new SQLServerParameterMetaData(this, userSQL); + + connection.cacheParameterMetadata(userSQL, pmd, this); + + loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); + + return pmd; + } + } + /* JDBC 3.0 */ /* L3 */ public final ParameterMetaData getParameterMetaData() throws SQLServerException { - loggerExternal.entering(getClassNameLogging(), "getParameterMetaData"); - checkClosed(); - SQLServerParameterMetaData pmd = new SQLServerParameterMetaData(this, userSQL); - loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); - return pmd; + return getParameterMetaData(false); } /* L3 */ public final void setURL(int parameterIndex, diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResource.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResource.java index 5d8c86d9b..b84796831 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResource.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResource.java @@ -190,6 +190,7 @@ protected Object[][] getContents() { {"R_socketTimeoutPropertyDescription", "The number of milliseconds to wait before the java.net.SocketTimeoutException is raised."}, {"R_serverPreparedStatementDiscardThresholdPropertyDescription", "The threshold for when to close discarded prepare statements on the server (calling a batch of sp_unprepares). A value of 1 or less will cause sp_unprepare to be called immediately on PreparedStatment close."}, {"R_enablePrepareOnFirstPreparedStatementCallPropertyDescription", "This setting specifies whether a prepared statement is prepared (sp_prepexec) on first use (property=true) or on second after first calling sp_executesql (property=false)."}, + {"R_statementPoolingCacheSizePropertyDescription", "This setting specifies the size of the prepared statement cache for a conection. A value less than 1 means no cache."}, {"R_gsscredentialPropertyDescription", "Impersonated GSS Credential to access SQL Server."}, {"R_noParserSupport", "An error occurred while instantiating the required parser. Error: \"{0}\""}, {"R_writeOnlyXML", "Cannot read from this SQLXML instance. This instance is for writing data only."}, @@ -380,6 +381,7 @@ protected Object[][] getContents() { {"R_invalidFipsEncryptConfig", "Could not enable FIPS due to either encrypt is not true or using trusted certificate settings."}, {"R_invalidFipsProviderConfig", "Could not enable FIPS due to invalid FIPSProvider or TrustStoreType."}, {"R_serverPreparedStatementDiscardThreshold", "The serverPreparedStatementDiscardThreshold {0} is not valid."}, + {"R_statementPoolingCacheSize", "The statementPoolingCacheSize {0} is not valid."}, {"R_kerberosLoginFailedForUsername", "Cannot login with Kerberos principal {0}, check your credentials. {1}"}, {"R_kerberosLoginFailed", "Kerberos Login failed: {0} due to {1} ({2})"}, }; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index d49d72221..63776dd64 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -761,10 +761,17 @@ final void processResponse(TDSReader tdsReader) throws SQLServerException { private String ensureSQLSyntax(String sql) throws SQLServerException { if (sql.indexOf(LEFT_CURLY_BRACKET) >= 0) { - JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); - String execSyntax = translator.translate(sql); - procedureName = translator.getProcedureName(); - return execSyntax; + + // Check for cached SQL metadata. + ParsedSQLCacheItem cacheItem = SQLServerPreparedStatement.getCachedParsedSQLMetadata(sql); + + // No cached SQL-text meta datafound, parse. + if(null == cacheItem) + cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql); + + // Retrieve from cache item. + procedureName = cacheItem.procedureName; + return cacheItem.preparedSQLText; } return sql; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 03ed1167c..3eab507ab 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -55,9 +55,6 @@ private int executeSQLReturnFirstInt(SQLServerConnection conn, String sql) throw public void testBatchedUnprepare() throws SQLException { SQLServerConnection conOuter = null; - // Turn off use of prepared statement cache. - SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 0; - // Make sure correct settings are used. SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); @@ -65,6 +62,9 @@ public void testBatchedUnprepare() throws SQLException { try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { conOuter = con; + // Turn off use of prepared statement cache. + con.setStatementPoolingCacheSize(0); + // Clean-up proc cache this.executeSQL(con, "DBCC FREEPROCCACHE;"); @@ -133,15 +133,70 @@ public void testBatchedUnprepare() throws SQLException { } /** - * Test handling of the two configuration knobs related to prepared statement handling. + * Test handling of statement pooling for prepared statements. * * @throws SQLException */ @Test - public void testPreparedStatementExecAndUnprepareConfig() throws SQLException { + public void testStatementPooling() throws SQLException { + // Make sure correct settings are used. + SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); + SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); + + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + + // Test behvaior with statement pooling. + con.setStatementPoolingCacheSize(10); + + String lookupUniqueifier = UUID.randomUUID().toString(); + String query = String.format("/*statementpoolingtest_%s*/SELECT * FROM sys.tables;", lookupUniqueifier); + + // Execute statement first, should create cache entry WITHOUT handle (since sp_executesql was used). + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_executesql + + assertSame(0, pstmt.getPreparedStatementHandle()); + } + + // Execute statement again, should now create handle. + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_prepexec + } + + // Execute statement again and save handle, should now have used handle from cache. + int handle = 0; + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_execute + + handle = pstmt.getPreparedStatementHandle(); + assertNotSame(0, pstmt.getPreparedStatementHandle()); + } + + // Execute statement again and verify same handle was used. + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_execute + + assertNotSame(0, pstmt.getPreparedStatementHandle()); + assertSame(handle, pstmt.getPreparedStatementHandle()); + } - // Turn off use of prepared statement cache. - SQLServerConnection.preparedStatementHandleCacheSize_SHOULD_BE_CONNECTION_STRING_PROPERTY = 0; + // Execute new statement with different SQL text and verify it does NOT get same handle (should now fall back to using sp_executesql). + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + ";")) { + pstmt.execute(); // sp_executesql + + assertSame(0, pstmt.getPreparedStatementHandle()); + assertNotSame(handle, pstmt.getPreparedStatementHandle()); + } + } + } + + /** + * Test handling of the two configuration knobs related to prepared statement handling. + * + * @throws SQLException + */ + @Test + public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws SQLException { // Verify initial defaults are correct: assertTrue(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold() > 1); @@ -153,15 +208,19 @@ public void testPreparedStatementExecAndUnprepareConfig() throws SQLException { SQLServerDataSource dataSource = new SQLServerDataSource(); dataSource.setURL(connectionString); // Verify defaults. + assertTrue(0 < dataSource.getStatementPoolingCacheSize()); assertSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), dataSource.getEnablePrepareOnFirstPreparedStatementCall()); assertSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), dataSource.getServerPreparedStatementDiscardThreshold()); // Verify change + dataSource.setStatementPoolingCacheSize(0); + assertSame(0, dataSource.getStatementPoolingCacheSize()); dataSource.setEnablePrepareOnFirstPreparedStatementCall(!dataSource.getEnablePrepareOnFirstPreparedStatementCall()); assertNotSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), dataSource.getEnablePrepareOnFirstPreparedStatementCall()); dataSource.setServerPreparedStatementDiscardThreshold(dataSource.getServerPreparedStatementDiscardThreshold() + 1); assertNotSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), dataSource.getServerPreparedStatementDiscardThreshold()); // Verify connection from data source has same parameters. SQLServerConnection connDataSource = (SQLServerConnection)dataSource.getConnection(); + assertSame(dataSource.getStatementPoolingCacheSize(), connDataSource.getStatementPoolingCacheSize()); assertSame(dataSource.getEnablePrepareOnFirstPreparedStatementCall(), connDataSource.getEnablePrepareOnFirstPreparedStatementCall()); assertSame(dataSource.getServerPreparedStatementDiscardThreshold(), connDataSource.getServerPreparedStatementDiscardThreshold()); @@ -170,6 +229,15 @@ public void testPreparedStatementExecAndUnprepareConfig() throws SQLException { assertNotSame(true, SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); assertNotSame(3, SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); + // Test disableStatementPooling + String connectionStringDisableStatementPooling = connectionString + ";disableStatementPooling=true;"; + SQLServerConnection connectionDisableStatementPooling = (SQLServerConnection)DriverManager.getConnection(connectionStringDisableStatementPooling); + assertSame(0, connectionDisableStatementPooling.getStatementPoolingCacheSize()); + assertTrue(!connectionDisableStatementPooling.isStatementPoolingEnabled()); + String connectionStringEnableStatementPooling = connectionString + ";disableStatementPooling=false;"; + SQLServerConnection connectionEnableStatementPooling = (SQLServerConnection)DriverManager.getConnection(connectionStringEnableStatementPooling); + assertTrue(0 < connectionEnableStatementPooling.getStatementPoolingCacheSize()); + // Test EnablePrepareOnFirstPreparedStatementCall String connectionStringNoExecuteSQL = connectionString + ";enablePrepareOnFirstPreparedStatementCall=true;"; SQLServerConnection connectionNoExecuteSQL = (SQLServerConnection)DriverManager.getConnection(connectionStringNoExecuteSQL); @@ -232,6 +300,9 @@ public void testPreparedStatementExecAndUnprepareConfig() throws SQLException { SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + // Turn off use of prepared statement cache. + con.setStatementPoolingCacheSize(0); + String query = "/*unprepSettingsTest*/SELECT * FROM sys.objects;"; // Verify initial default is not serial: From f896542fd3a37de23f976ebca01ec40aa2d0a0ab Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Wed, 3 May 2017 23:33:26 -0500 Subject: [PATCH 08/34] Fix re-use of parameter metadata with closed parent statement. --- .../microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java index c0e26d646..b779a735e 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java @@ -551,7 +551,9 @@ String parseThreePartNames(String threeName) throws SQLServerException { } private void checkClosed() throws SQLServerException { - stmtParent.checkClosed(); + // stmtParent does not seem to be re-used, should just verify connection is not closed. + // stmtParent.checkClosed(); + con.checkClosed(); } /** From dab1c6bf93a572dbcd7f9582d786acc7238e758b Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Thu, 4 May 2017 12:32:06 -0500 Subject: [PATCH 09/34] SHA1 hash of SQL as cache key --- ...SQLCacheItem.java => ParsedSQLCacheItem.java} | 16 ++++++++++++++++ .../sqlserver/jdbc/SQLServerConnection.java | 16 +++++++++++----- .../jdbc/SQLServerPreparedStatement.java | 13 ++++++++++--- 3 files changed, 37 insertions(+), 8 deletions(-) rename src/main/java/com/microsoft/sqlserver/jdbc/{SQLServerParsedSQLCacheItem.java => ParsedSQLCacheItem.java} (69%) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java similarity index 69% rename from src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java rename to src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java index 0ae2db889..8647bb610 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java @@ -8,6 +8,9 @@ package com.microsoft.sqlserver.jdbc; +import org.apache.commons.codec.digest.DigestUtils; +import java.nio.ByteBuffer; + /** * Used to keep track of parsed SQL text and its properties for prepared statements. */ @@ -24,4 +27,17 @@ final class ParsedSQLCacheItem { this.procedureName = procedureName; this.bReturnValueSyntax = bReturnValueSyntax; } + + static ByteBuffer generateHash(String originalKey) { + try { + if(null == originalKey) + return null; + else + return ByteBuffer.wrap(DigestUtils.sha1(originalKey)); + } + catch(Exception e) { + return null; + } + } } + diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index b64fb331e..002bdf386 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -18,6 +18,7 @@ import java.net.InetAddress; import java.net.SocketException; import java.net.UnknownHostException; +import java.security.NoSuchAlgorithmException; import java.sql.Blob; import java.sql.CallableStatement; import java.sql.Clob; @@ -49,6 +50,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.logging.Level; +import java.nio.ByteBuffer; import javax.sql.XAConnection; import javax.xml.bind.DatatypeConverter; @@ -181,7 +183,7 @@ void decrementHandleRefCount() { private int statementPoolingCacheSize = 10; /** Cache of prepared statement handles */ - private Cache preparedStatementCache; + private Cache preparedStatementCache; SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; @@ -5594,9 +5596,11 @@ final PreparedStatementCacheItem getCachedPreparedStatementMetadata(String sql) if(null == this.preparedStatementCache) return null; - PreparedStatementCacheItem cacheItem = this.preparedStatementCache.getIfPresent(sql); - - return cacheItem; + ByteBuffer key = ParsedSQLCacheItem.generateHash(sql); + if(null == key) + return null; + + return this.preparedStatementCache.getIfPresent(key); } // Handle closing handles when removed from cache. @@ -5677,7 +5681,9 @@ private void cachePreparedStatementMetadata(String sql, PreparedStatementCacheIt .build(); } - this.preparedStatementCache.put(sql, cacheItem); + ByteBuffer key = ParsedSQLCacheItem.generateHash(sql); + if(null != key) + this.preparedStatementCache.put(key, cacheItem); } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index d1d42cbe2..2de386b90 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Vector; import java.util.logging.Level; +import java.nio.ByteBuffer; import com.google.common.cache.CacheBuilder; import com.google.common.cache.Cache; @@ -148,7 +149,7 @@ private void resetPrepStmtHandle() { static final private int parsedSQLCacheSize = 100; /** Cache of prepared statement meta data */ - static private Cache parsedSQLCache; + static private Cache parsedSQLCache; static { parsedSQLCache = CacheBuilder.newBuilder() .maximumSize(parsedSQLCacheSize) @@ -157,7 +158,11 @@ private void resetPrepStmtHandle() { /** Get prepared statement cache entry if exists */ static ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { - return parsedSQLCache.getIfPresent(initialSql); + ByteBuffer key = ParsedSQLCacheItem.generateHash(initialSql); + if(null == key) + return null; + else + return parsedSQLCache.getIfPresent(initialSql); } /** Add cache entry for prepared statement metadata*/ @@ -172,7 +177,9 @@ static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql) throws SQL // Cache this entry. ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); - parsedSQLCache.put(initialSql, cacheItem); + ByteBuffer key = ParsedSQLCacheItem.generateHash(initialSql); + if(null != key) + parsedSQLCache.put(key, cacheItem); return cacheItem; } From dbc95229d0eea05b90da57580ede68bb22f0af20 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Thu, 4 May 2017 17:00:19 -0500 Subject: [PATCH 10/34] Undo use of hash for statement pooling key. --- .../sqlserver/jdbc/SQLServerConnection.java | 16 ++++++---------- .../jdbc/SQLServerPreparedStatement.java | 11 ++++------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 002bdf386..1858b04fc 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -18,7 +18,6 @@ import java.net.InetAddress; import java.net.SocketException; import java.net.UnknownHostException; -import java.security.NoSuchAlgorithmException; import java.sql.Blob; import java.sql.CallableStatement; import java.sql.Clob; @@ -50,7 +49,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.logging.Level; -import java.nio.ByteBuffer; import javax.sql.XAConnection; import javax.xml.bind.DatatypeConverter; @@ -183,7 +181,7 @@ void decrementHandleRefCount() { private int statementPoolingCacheSize = 10; /** Cache of prepared statement handles */ - private Cache preparedStatementCache; + private Cache preparedStatementCache; SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; @@ -5596,11 +5594,10 @@ final PreparedStatementCacheItem getCachedPreparedStatementMetadata(String sql) if(null == this.preparedStatementCache) return null; - ByteBuffer key = ParsedSQLCacheItem.generateHash(sql); - if(null == key) + if(null == sql) return null; - - return this.preparedStatementCache.getIfPresent(key); + else + return this.preparedStatementCache.getIfPresent(sql); } // Handle closing handles when removed from cache. @@ -5681,9 +5678,8 @@ private void cachePreparedStatementMetadata(String sql, PreparedStatementCacheIt .build(); } - ByteBuffer key = ParsedSQLCacheItem.generateHash(sql); - if(null != key) - this.preparedStatementCache.put(key, cacheItem); + if(null != sql) + this.preparedStatementCache.put(sql, cacheItem); } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 2de386b90..d111d41d7 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Vector; import java.util.logging.Level; -import java.nio.ByteBuffer; import com.google.common.cache.CacheBuilder; import com.google.common.cache.Cache; @@ -149,7 +148,7 @@ private void resetPrepStmtHandle() { static final private int parsedSQLCacheSize = 100; /** Cache of prepared statement meta data */ - static private Cache parsedSQLCache; + static private Cache parsedSQLCache; static { parsedSQLCache = CacheBuilder.newBuilder() .maximumSize(parsedSQLCacheSize) @@ -158,8 +157,7 @@ private void resetPrepStmtHandle() { /** Get prepared statement cache entry if exists */ static ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { - ByteBuffer key = ParsedSQLCacheItem.generateHash(initialSql); - if(null == key) + if(null == initialSql) return null; else return parsedSQLCache.getIfPresent(initialSql); @@ -177,9 +175,8 @@ static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql) throws SQL // Cache this entry. ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); - ByteBuffer key = ParsedSQLCacheItem.generateHash(initialSql); - if(null != key) - parsedSQLCache.put(key, cacheItem); + if(null != initialSql) + parsedSQLCache.put(initialSql, cacheItem); return cacheItem; } From 0e8b9aab51a631e9e676d0f7367070daec21978c Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Fri, 5 May 2017 17:03:03 -0500 Subject: [PATCH 11/34] Replaced Guava with CLHM + SHA1 hash --- build.gradle | 3 +- pom.xml | 7 - .../ConcurrentLinkedHashMap.java | 1574 +++++++++++++++++ .../concurrentlinkedhashmap/EntryWeigher.java | 37 + .../EvictionListener.java | 45 + .../concurrentlinkedhashmap/LICENSE | 201 +++ .../concurrentlinkedhashmap/LinkedDeque.java | 460 +++++ .../googlecode/concurrentlinkedhashmap/NOTICE | 7 + .../concurrentlinkedhashmap/Weigher.java | 36 + .../concurrentlinkedhashmap/Weighers.java | 282 +++ .../concurrentlinkedhashmap/package-info.java | 41 + ...CacheItem.java => ParsedSQLCacheItem.java} | 1 + .../sqlserver/jdbc/SQLServerConnection.java | 72 +- .../jdbc/SQLServerParameterMetaData.java | 4 +- .../jdbc/SQLServerPreparedStatement.java | 98 +- .../sqlserver/jdbc/SQLServerStatement.java | 6 +- .../unit/statement/PreparedStatementTest.java | 14 +- 17 files changed, 2806 insertions(+), 82 deletions(-) create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java create mode 100644 src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java rename src/main/java/com/microsoft/sqlserver/jdbc/{SQLServerParsedSQLCacheItem.java => ParsedSQLCacheItem.java} (99%) diff --git a/build.gradle b/build.gradle index 38e407d03..7a402dfbc 100644 --- a/build.gradle +++ b/build.gradle @@ -67,8 +67,7 @@ repositories { dependencies { compile 'com.microsoft.azure:azure-keyvault:0.9.7', - 'com.microsoft.azure:adal4j:1.1.3', - 'com.google.guava:guava:19.0' + 'com.microsoft.azure:adal4j:1.1.3' testCompile 'junit:junit:4.12', 'org.junit.platform:junit-platform-console:1.0.0-M3', diff --git a/pom.xml b/pom.xml index 6268b8be2..c5d123987 100644 --- a/pom.xml +++ b/pom.xml @@ -59,13 +59,6 @@ true - - com.google.guava - guava - 19.0 - false - - junit diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java new file mode 100644 index 000000000..93079f93e --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java @@ -0,0 +1,1574 @@ +/* + * Copyright 2010 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.IDLE; +import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.PROCESSING; +import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.REQUIRED; +import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; + +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.AbstractCollection; +import java.util.AbstractMap; +import java.util.AbstractQueue; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * A hash table supporting full concurrency of retrievals, adjustable expected + * concurrency for updates, and a maximum capacity to bound the map by. This + * implementation differs from {@link ConcurrentHashMap} in that it maintains a + * page replacement algorithm that is used to evict an entry when the map has + * exceeded its capacity. Unlike the Java Collections Framework, this + * map does not have a publicly visible constructor and instances are created + * through a {@link Builder}. + *

+ * An entry is evicted from the map when the weighted capacity exceeds + * its maximum weighted capacity threshold. A {@link EntryWeigher} + * determines how many units of capacity that an entry consumes. The default + * weigher assigns each value a weight of 1 to bound the map by the + * total number of key-value pairs. A map that holds collections may choose to + * weigh values by the number of elements in the collection and bound the map + * by the total number of elements that it contains. A change to a value that + * modifies its weight requires that an update operation is performed on the + * map. + *

+ * An {@link EvictionListener} may be supplied for notification when an entry + * is evicted from the map. This listener is invoked on a caller's thread and + * will not block other threads from operating on the map. An implementation + * should be aware that the caller's thread will not expect long execution + * times or failures as a side effect of the listener being notified. Execution + * safety and a fast turn around time can be achieved by performing the + * operation asynchronously, such as by submitting a task to an + * {@link java.util.concurrent.ExecutorService}. + *

+ * The concurrency level determines the number of threads that can + * concurrently modify the table. Using a significantly higher or lower value + * than needed can waste space or lead to thread contention, but an estimate + * within an order of magnitude of the ideal value does not usually have a + * noticeable impact. Because placement in hash tables is essentially random, + * the actual concurrency will vary. + *

+ * This class and its views and iterators implement all of the + * optional methods of the {@link Map} and {@link Iterator} + * interfaces. + *

+ * Like {@link java.util.Hashtable} but unlike {@link HashMap}, this class + * does not allow null to be used as a key or value. Unlike + * {@link java.util.LinkedHashMap}, this class does not provide + * predictable iteration order. A snapshot of the keys and entries may be + * obtained in ascending and descending order of retention. + * + * @author ben.manes@gmail.com (Ben Manes) + * @param the type of keys maintained by this map + * @param the type of mapped values + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +public final class ConcurrentLinkedHashMap extends AbstractMap + implements ConcurrentMap, Serializable { + + /* + * This class performs a best-effort bounding of a ConcurrentHashMap using a + * page-replacement algorithm to determine which entries to evict when the + * capacity is exceeded. + * + * The page replacement algorithm's data structures are kept eventually + * consistent with the map. An update to the map and recording of reads may + * not be immediately reflected on the algorithm's data structures. These + * structures are guarded by a lock and operations are applied in batches to + * avoid lock contention. The penalty of applying the batches is spread across + * threads so that the amortized cost is slightly higher than performing just + * the ConcurrentHashMap operation. + * + * A memento of the reads and writes that were performed on the map are + * recorded in buffers. These buffers are drained at the first opportunity + * after a write or when the read buffer exceeds a threshold size. The reads + * are recorded in a lossy buffer, allowing the reordering operations to be + * discarded if the draining process cannot keep up. Due to the concurrent + * nature of the read and write operations a strict policy ordering is not + * possible, but is observably strict when single threaded. + * + * Due to a lack of a strict ordering guarantee, a task can be executed + * out-of-order, such as a removal followed by its addition. The state of the + * entry is encoded within the value's weight. + * + * Alive: The entry is in both the hash-table and the page replacement policy. + * This is represented by a positive weight. + * + * Retired: The entry is not in the hash-table and is pending removal from the + * page replacement policy. This is represented by a negative weight. + * + * Dead: The entry is not in the hash-table and is not in the page replacement + * policy. This is represented by a weight of zero. + * + * The Least Recently Used page replacement algorithm was chosen due to its + * simplicity, high hit rate, and ability to be implemented with O(1) time + * complexity. + */ + + /** The number of CPUs */ + static final int NCPU = Runtime.getRuntime().availableProcessors(); + + /** The maximum weighted capacity of the map. */ + static final long MAXIMUM_CAPACITY = Long.MAX_VALUE - Integer.MAX_VALUE; + + /** The number of read buffers to use. */ + static final int NUMBER_OF_READ_BUFFERS = ceilingNextPowerOfTwo(NCPU); + + /** Mask value for indexing into the read buffers. */ + static final int READ_BUFFERS_MASK = NUMBER_OF_READ_BUFFERS - 1; + + /** The number of pending read operations before attempting to drain. */ + static final int READ_BUFFER_THRESHOLD = 32; + + /** The maximum number of read operations to perform per amortized drain. */ + static final int READ_BUFFER_DRAIN_THRESHOLD = 2 * READ_BUFFER_THRESHOLD; + + /** The maximum number of pending reads per buffer. */ + static final int READ_BUFFER_SIZE = 2 * READ_BUFFER_DRAIN_THRESHOLD; + + /** Mask value for indexing into the read buffer. */ + static final int READ_BUFFER_INDEX_MASK = READ_BUFFER_SIZE - 1; + + /** The maximum number of write operations to perform per amortized drain. */ + static final int WRITE_BUFFER_DRAIN_THRESHOLD = 16; + + /** A queue that discards all entries. */ + static final Queue DISCARDING_QUEUE = new DiscardingQueue(); + + static int ceilingNextPowerOfTwo(int x) { + // From Hacker's Delight, Chapter 3, Harry S. Warren Jr. + return 1 << (Integer.SIZE - Integer.numberOfLeadingZeros(x - 1)); + } + + // The backing data store holding the key-value associations + final ConcurrentMap> data; + final int concurrencyLevel; + + // These fields provide support to bound the map by a maximum capacity + final long[] readBufferReadCount; + final LinkedDeque> evictionDeque; + + final AtomicLong weightedSize; + final AtomicLong capacity; + + final Lock evictionLock; + final Queue writeBuffer; + final AtomicLong[] readBufferWriteCount; + final AtomicLong[] readBufferDrainAtWriteCount; + final AtomicReference>[][] readBuffers; + + final AtomicReference drainStatus; + final EntryWeigher weigher; + + // These fields provide support for notifying a listener. + final Queue> pendingNotifications; + final EvictionListener listener; + + transient Set keySet; + transient Collection values; + transient Set> entrySet; + + /** + * Creates an instance based on the builder's configuration. + */ + @SuppressWarnings({"unchecked", "cast"}) + private ConcurrentLinkedHashMap(Builder builder) { + // The data store and its maximum capacity + concurrencyLevel = builder.concurrencyLevel; + capacity = new AtomicLong(Math.min(builder.capacity, MAXIMUM_CAPACITY)); + data = new ConcurrentHashMap>(builder.initialCapacity, 0.75f, concurrencyLevel); + + // The eviction support + weigher = builder.weigher; + evictionLock = new ReentrantLock(); + weightedSize = new AtomicLong(); + evictionDeque = new LinkedDeque>(); + writeBuffer = new ConcurrentLinkedQueue(); + drainStatus = new AtomicReference(IDLE); + + readBufferReadCount = new long[NUMBER_OF_READ_BUFFERS]; + readBufferWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS]; + readBufferDrainAtWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS]; + readBuffers = new AtomicReference[NUMBER_OF_READ_BUFFERS][READ_BUFFER_SIZE]; + for (int i = 0; i < NUMBER_OF_READ_BUFFERS; i++) { + readBufferWriteCount[i] = new AtomicLong(); + readBufferDrainAtWriteCount[i] = new AtomicLong(); + readBuffers[i] = new AtomicReference[READ_BUFFER_SIZE]; + for (int j = 0; j < READ_BUFFER_SIZE; j++) { + readBuffers[i][j] = new AtomicReference>(); + } + } + + // The notification queue and listener + listener = builder.listener; + pendingNotifications = (listener == DiscardingListener.INSTANCE) + ? (Queue>) DISCARDING_QUEUE + : new ConcurrentLinkedQueue>(); + } + + /** Ensures that the object is not null. */ + static void checkNotNull(Object o) { + if (o == null) { + throw new NullPointerException(); + } + } + + /** Ensures that the argument expression is true. */ + static void checkArgument(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } + + /** Ensures that the state expression is true. */ + static void checkState(boolean expression) { + if (!expression) { + throw new IllegalStateException(); + } + } + + /* ---------------- Eviction Support -------------- */ + + /** + * Retrieves the maximum weighted capacity of the map. + * + * @return the maximum weighted capacity + */ + public long capacity() { + return capacity.get(); + } + + /** + * Sets the maximum weighted capacity of the map and eagerly evicts entries + * until it shrinks to the appropriate size. + * + * @param capacity the maximum weighted capacity of the map + * @throws IllegalArgumentException if the capacity is negative + */ + public void setCapacity(long capacity) { + checkArgument(capacity >= 0); + evictionLock.lock(); + try { + this.capacity.lazySet(Math.min(capacity, MAXIMUM_CAPACITY)); + drainBuffers(); + evict(); + } finally { + evictionLock.unlock(); + } + notifyListener(); + } + + /** Determines whether the map has exceeded its capacity. */ + boolean hasOverflowed() { + return weightedSize.get() > capacity.get(); + } + + /** + * Evicts entries from the map while it exceeds the capacity and appends + * evicted entries to the notification queue for processing. + */ + void evict() { + // Attempts to evict entries from the map if it exceeds the maximum + // capacity. If the eviction fails due to a concurrent removal of the + // victim, that removal may cancel out the addition that triggered this + // eviction. The victim is eagerly unlinked before the removal task so + // that if an eviction is still required then a new victim will be chosen + // for removal. + while (hasOverflowed()) { + final Node node = evictionDeque.poll(); + + // If weighted values are used, then the pending operations will adjust + // the size to reflect the correct weight + if (node == null) { + return; + } + + // Notify the listener only if the entry was evicted + if (data.remove(node.key, node)) { + pendingNotifications.add(node); + } + + makeDead(node); + } + } + + /** + * Performs the post-processing work required after a read. + * + * @param node the entry in the page replacement policy + */ + void afterRead(Node node) { + final int bufferIndex = readBufferIndex(); + final long writeCount = recordRead(bufferIndex, node); + drainOnReadIfNeeded(bufferIndex, writeCount); + notifyListener(); + } + + /** Returns the index to the read buffer to record into. */ + static int readBufferIndex() { + // A buffer is chosen by the thread's id so that tasks are distributed in a + // pseudo evenly manner. This helps avoid hot entries causing contention + // due to other threads trying to append to the same buffer. + return ((int) Thread.currentThread().getId()) & READ_BUFFERS_MASK; + } + + /** + * Records a read in the buffer and return its write count. + * + * @param bufferIndex the index to the chosen read buffer + * @param node the entry in the page replacement policy + * @return the number of writes on the chosen read buffer + */ + long recordRead(int bufferIndex, Node node) { + // The location in the buffer is chosen in a racy fashion as the increment + // is not atomic with the insertion. This means that concurrent reads can + // overlap and overwrite one another, resulting in a lossy buffer. + final AtomicLong counter = readBufferWriteCount[bufferIndex]; + final long writeCount = counter.get(); + counter.lazySet(writeCount + 1); + + final int index = (int) (writeCount & READ_BUFFER_INDEX_MASK); + readBuffers[bufferIndex][index].lazySet(node); + + return writeCount; + } + + /** + * Attempts to drain the buffers if it is determined to be needed when + * post-processing a read. + * + * @param bufferIndex the index to the chosen read buffer + * @param writeCount the number of writes on the chosen read buffer + */ + void drainOnReadIfNeeded(int bufferIndex, long writeCount) { + final long pending = (writeCount - readBufferDrainAtWriteCount[bufferIndex].get()); + final boolean delayable = (pending < READ_BUFFER_THRESHOLD); + final DrainStatus status = drainStatus.get(); + if (status.shouldDrainBuffers(delayable)) { + tryToDrainBuffers(); + } + } + + /** + * Performs the post-processing work required after a write. + * + * @param task the pending operation to be applied + */ + void afterWrite(Runnable task) { + writeBuffer.add(task); + drainStatus.lazySet(REQUIRED); + tryToDrainBuffers(); + notifyListener(); + } + + /** + * Attempts to acquire the eviction lock and apply the pending operations, up + * to the amortized threshold, to the page replacement policy. + */ + void tryToDrainBuffers() { + if (evictionLock.tryLock()) { + try { + drainStatus.lazySet(PROCESSING); + drainBuffers(); + } finally { + drainStatus.compareAndSet(PROCESSING, IDLE); + evictionLock.unlock(); + } + } + } + + /** Drains the read and write buffers up to an amortized threshold. */ + void drainBuffers() { + drainReadBuffers(); + drainWriteBuffer(); + } + + /** Drains the read buffers, each up to an amortized threshold. */ + void drainReadBuffers() { + final int start = (int) Thread.currentThread().getId(); + final int end = start + NUMBER_OF_READ_BUFFERS; + for (int i = start; i < end; i++) { + drainReadBuffer(i & READ_BUFFERS_MASK); + } + } + + /** Drains the read buffer up to an amortized threshold. */ + void drainReadBuffer(int bufferIndex) { + final long writeCount = readBufferWriteCount[bufferIndex].get(); + for (int i = 0; i < READ_BUFFER_DRAIN_THRESHOLD; i++) { + final int index = (int) (readBufferReadCount[bufferIndex] & READ_BUFFER_INDEX_MASK); + final AtomicReference> slot = readBuffers[bufferIndex][index]; + final Node node = slot.get(); + if (node == null) { + break; + } + + slot.lazySet(null); + applyRead(node); + readBufferReadCount[bufferIndex]++; + } + readBufferDrainAtWriteCount[bufferIndex].lazySet(writeCount); + } + + /** Updates the node's location in the page replacement policy. */ + void applyRead(Node node) { + // An entry may be scheduled for reordering despite having been removed. + // This can occur when the entry was concurrently read while a writer was + // removing it. If the entry is no longer linked then it does not need to + // be processed. + if (evictionDeque.contains(node)) { + evictionDeque.moveToBack(node); + } + } + + /** Drains the read buffer up to an amortized threshold. */ + void drainWriteBuffer() { + for (int i = 0; i < WRITE_BUFFER_DRAIN_THRESHOLD; i++) { + final Runnable task = writeBuffer.poll(); + if (task == null) { + break; + } + task.run(); + } + } + + /** + * Attempts to transition the node from the alive state to the + * retired state. + * + * @param node the entry in the page replacement policy + * @param expect the expected weighted value + * @return if successful + */ + boolean tryToRetire(Node node, WeightedValue expect) { + if (expect.isAlive()) { + final WeightedValue retired = new WeightedValue(expect.value, -expect.weight); + return node.compareAndSet(expect, retired); + } + return false; + } + + /** + * Atomically transitions the node from the alive state to the + * retired state, if a valid transition. + * + * @param node the entry in the page replacement policy + */ + void makeRetired(Node node) { + for (;;) { + final WeightedValue current = node.get(); + if (!current.isAlive()) { + return; + } + final WeightedValue retired = new WeightedValue(current.value, -current.weight); + if (node.compareAndSet(current, retired)) { + return; + } + } + } + + /** + * Atomically transitions the node to the dead state and decrements + * the weightedSize. + * + * @param node the entry in the page replacement policy + */ + void makeDead(Node node) { + for (;;) { + WeightedValue current = node.get(); + WeightedValue dead = new WeightedValue(current.value, 0); + if (node.compareAndSet(current, dead)) { + weightedSize.lazySet(weightedSize.get() - Math.abs(current.weight)); + return; + } + } + } + + /** Notifies the listener of entries that were evicted. */ + void notifyListener() { + Node node; + while ((node = pendingNotifications.poll()) != null) { + listener.onEviction(node.key, node.getValue()); + } + } + + /** Adds the node to the page replacement policy. */ + final class AddTask implements Runnable { + final Node node; + final int weight; + + AddTask(Node node, int weight) { + this.weight = weight; + this.node = node; + } + + @Override + public void run() { + weightedSize.lazySet(weightedSize.get() + weight); + + // ignore out-of-order write operations + if (node.get().isAlive()) { + evictionDeque.add(node); + evict(); + } + } + } + + /** Removes a node from the page replacement policy. */ + final class RemovalTask implements Runnable { + final Node node; + + RemovalTask(Node node) { + this.node = node; + } + + @Override + public void run() { + // add may not have been processed yet + evictionDeque.remove(node); + makeDead(node); + } + } + + /** Updates the weighted size and evicts an entry on overflow. */ + final class UpdateTask implements Runnable { + final int weightDifference; + final Node node; + + public UpdateTask(Node node, int weightDifference) { + this.weightDifference = weightDifference; + this.node = node; + } + + @Override + public void run() { + weightedSize.lazySet(weightedSize.get() + weightDifference); + applyRead(node); + evict(); + } + } + + /* ---------------- Concurrent Map Support -------------- */ + + @Override + public boolean isEmpty() { + return data.isEmpty(); + } + + @Override + public int size() { + return data.size(); + } + + /** + * Returns the weighted size of this map. + * + * @return the combined weight of the values in this map + */ + public long weightedSize() { + return Math.max(0, weightedSize.get()); + } + + @Override + public void clear() { + evictionLock.lock(); + try { + // Discard all entries + Node node; + while ((node = evictionDeque.poll()) != null) { + data.remove(node.key, node); + makeDead(node); + } + + // Discard all pending reads + for (AtomicReference>[] buffer : readBuffers) { + for (AtomicReference> slot : buffer) { + slot.lazySet(null); + } + } + + // Apply all pending writes + Runnable task; + while ((task = writeBuffer.poll()) != null) { + task.run(); + } + } finally { + evictionLock.unlock(); + } + } + + @Override + public boolean containsKey(Object key) { + return data.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + checkNotNull(value); + + for (Node node : data.values()) { + if (node.getValue().equals(value)) { + return true; + } + } + return false; + } + + @Override + public V get(Object key) { + final Node node = data.get(key); + if (node == null) { + return null; + } + afterRead(node); + return node.getValue(); + } + + /** + * Returns the value to which the specified key is mapped, or {@code null} + * if this map contains no mapping for the key. This method differs from + * {@link #get(Object)} in that it does not record the operation with the + * page replacement policy. + * + * @param key the key whose associated value is to be returned + * @return the value to which the specified key is mapped, or + * {@code null} if this map contains no mapping for the key + * @throws NullPointerException if the specified key is null + */ + public V getQuietly(Object key) { + final Node node = data.get(key); + return (node == null) ? null : node.getValue(); + } + + @Override + public V put(K key, V value) { + return put(key, value, false); + } + + @Override + public V putIfAbsent(K key, V value) { + return put(key, value, true); + } + + /** + * Adds a node to the list and the data store. If an existing node is found, + * then its value is updated if allowed. + * + * @param key key with which the specified value is to be associated + * @param value value to be associated with the specified key + * @param onlyIfAbsent a write is performed only if the key is not already + * associated with a value + * @return the prior value in the data store or null if no mapping was found + */ + V put(K key, V value, boolean onlyIfAbsent) { + checkNotNull(key); + checkNotNull(value); + + final int weight = weigher.weightOf(key, value); + final WeightedValue weightedValue = new WeightedValue(value, weight); + final Node node = new Node(key, weightedValue); + + for (;;) { + final Node prior = data.putIfAbsent(node.key, node); + if (prior == null) { + afterWrite(new AddTask(node, weight)); + return null; + } else if (onlyIfAbsent) { + afterRead(prior); + return prior.getValue(); + } + for (;;) { + final WeightedValue oldWeightedValue = prior.get(); + if (!oldWeightedValue.isAlive()) { + break; + } + + if (prior.compareAndSet(oldWeightedValue, weightedValue)) { + final int weightedDifference = weight - oldWeightedValue.weight; + if (weightedDifference == 0) { + afterRead(prior); + } else { + afterWrite(new UpdateTask(prior, weightedDifference)); + } + return oldWeightedValue.value; + } + } + } + } + + @Override + public V remove(Object key) { + final Node node = data.remove(key); + if (node == null) { + return null; + } + + makeRetired(node); + afterWrite(new RemovalTask(node)); + return node.getValue(); + } + + @Override + public boolean remove(Object key, Object value) { + final Node node = data.get(key); + if ((node == null) || (value == null)) { + return false; + } + + WeightedValue weightedValue = node.get(); + for (;;) { + if (weightedValue.contains(value)) { + if (tryToRetire(node, weightedValue)) { + if (data.remove(key, node)) { + afterWrite(new RemovalTask(node)); + return true; + } + } else { + weightedValue = node.get(); + if (weightedValue.isAlive()) { + // retry as an intermediate update may have replaced the value with + // an equal instance that has a different reference identity + continue; + } + } + } + return false; + } + } + + @Override + public V replace(K key, V value) { + checkNotNull(key); + checkNotNull(value); + + final int weight = weigher.weightOf(key, value); + final WeightedValue weightedValue = new WeightedValue(value, weight); + + final Node node = data.get(key); + if (node == null) { + return null; + } + for (;;) { + final WeightedValue oldWeightedValue = node.get(); + if (!oldWeightedValue.isAlive()) { + return null; + } + if (node.compareAndSet(oldWeightedValue, weightedValue)) { + final int weightedDifference = weight - oldWeightedValue.weight; + if (weightedDifference == 0) { + afterRead(node); + } else { + afterWrite(new UpdateTask(node, weightedDifference)); + } + return oldWeightedValue.value; + } + } + } + + @Override + public boolean replace(K key, V oldValue, V newValue) { + checkNotNull(key); + checkNotNull(oldValue); + checkNotNull(newValue); + + final int weight = weigher.weightOf(key, newValue); + final WeightedValue newWeightedValue = new WeightedValue(newValue, weight); + + final Node node = data.get(key); + if (node == null) { + return false; + } + for (;;) { + final WeightedValue weightedValue = node.get(); + if (!weightedValue.isAlive() || !weightedValue.contains(oldValue)) { + return false; + } + if (node.compareAndSet(weightedValue, newWeightedValue)) { + final int weightedDifference = weight - weightedValue.weight; + if (weightedDifference == 0) { + afterRead(node); + } else { + afterWrite(new UpdateTask(node, weightedDifference)); + } + return true; + } + } + } + + @Override + public Set keySet() { + final Set ks = keySet; + return (ks == null) ? (keySet = new KeySet()) : ks; + } + + /** + * Returns a unmodifiable snapshot {@link Set} view of the keys contained in + * this map. The set's iterator returns the keys whose order of iteration is + * the ascending order in which its entries are considered eligible for + * retention, from the least-likely to be retained to the most-likely. + *

+ * Beware that, unlike in {@link #keySet()}, obtaining the set is NOT + * a constant-time operation. Because of the asynchronous nature of the page + * replacement policy, determining the retention ordering requires a traversal + * of the keys. + * + * @return an ascending snapshot view of the keys in this map + */ + public Set ascendingKeySet() { + return ascendingKeySetWithLimit(Integer.MAX_VALUE); + } + + /** + * Returns an unmodifiable snapshot {@link Set} view of the keys contained in + * this map. The set's iterator returns the keys whose order of iteration is + * the ascending order in which its entries are considered eligible for + * retention, from the least-likely to be retained to the most-likely. + *

+ * Beware that, unlike in {@link #keySet()}, obtaining the set is NOT + * a constant-time operation. Because of the asynchronous nature of the page + * replacement policy, determining the retention ordering requires a traversal + * of the keys. + * + * @param limit the maximum size of the returned set + * @return a ascending snapshot view of the keys in this map + * @throws IllegalArgumentException if the limit is negative + */ + public Set ascendingKeySetWithLimit(int limit) { + return orderedKeySet(true, limit); + } + + /** + * Returns an unmodifiable snapshot {@link Set} view of the keys contained in + * this map. The set's iterator returns the keys whose order of iteration is + * the descending order in which its entries are considered eligible for + * retention, from the most-likely to be retained to the least-likely. + *

+ * Beware that, unlike in {@link #keySet()}, obtaining the set is NOT + * a constant-time operation. Because of the asynchronous nature of the page + * replacement policy, determining the retention ordering requires a traversal + * of the keys. + * + * @return a descending snapshot view of the keys in this map + */ + public Set descendingKeySet() { + return descendingKeySetWithLimit(Integer.MAX_VALUE); + } + + /** + * Returns an unmodifiable snapshot {@link Set} view of the keys contained in + * this map. The set's iterator returns the keys whose order of iteration is + * the descending order in which its entries are considered eligible for + * retention, from the most-likely to be retained to the least-likely. + *

+ * Beware that, unlike in {@link #keySet()}, obtaining the set is NOT + * a constant-time operation. Because of the asynchronous nature of the page + * replacement policy, determining the retention ordering requires a traversal + * of the keys. + * + * @param limit the maximum size of the returned set + * @return a descending snapshot view of the keys in this map + * @throws IllegalArgumentException if the limit is negative + */ + public Set descendingKeySetWithLimit(int limit) { + return orderedKeySet(false, limit); + } + + Set orderedKeySet(boolean ascending, int limit) { + checkArgument(limit >= 0); + evictionLock.lock(); + try { + drainBuffers(); + + final int initialCapacity = (weigher == Weighers.entrySingleton()) + ? Math.min(limit, (int) weightedSize()) + : 16; + final Set keys = new LinkedHashSet(initialCapacity); + final Iterator> iterator = ascending + ? evictionDeque.iterator() + : evictionDeque.descendingIterator(); + while (iterator.hasNext() && (limit > keys.size())) { + keys.add(iterator.next().key); + } + return unmodifiableSet(keys); + } finally { + evictionLock.unlock(); + } + } + + @Override + public Collection values() { + final Collection vs = values; + return (vs == null) ? (values = new Values()) : vs; + } + + @Override + public Set> entrySet() { + final Set> es = entrySet; + return (es == null) ? (entrySet = new EntrySet()) : es; + } + + /** + * Returns an unmodifiable snapshot {@link Map} view of the mappings contained + * in this map. The map's collections return the mappings whose order of + * iteration is the ascending order in which its entries are considered + * eligible for retention, from the least-likely to be retained to the + * most-likely. + *

+ * Beware that obtaining the mappings is NOT a constant-time + * operation. Because of the asynchronous nature of the page replacement + * policy, determining the retention ordering requires a traversal of the + * entries. + * + * @return a ascending snapshot view of this map + */ + public Map ascendingMap() { + return ascendingMapWithLimit(Integer.MAX_VALUE); + } + + /** + * Returns an unmodifiable snapshot {@link Map} view of the mappings contained + * in this map. The map's collections return the mappings whose order of + * iteration is the ascending order in which its entries are considered + * eligible for retention, from the least-likely to be retained to the + * most-likely. + *

+ * Beware that obtaining the mappings is NOT a constant-time + * operation. Because of the asynchronous nature of the page replacement + * policy, determining the retention ordering requires a traversal of the + * entries. + * + * @param limit the maximum size of the returned map + * @return a ascending snapshot view of this map + * @throws IllegalArgumentException if the limit is negative + */ + public Map ascendingMapWithLimit(int limit) { + return orderedMap(true, limit); + } + + /** + * Returns an unmodifiable snapshot {@link Map} view of the mappings contained + * in this map. The map's collections return the mappings whose order of + * iteration is the descending order in which its entries are considered + * eligible for retention, from the most-likely to be retained to the + * least-likely. + *

+ * Beware that obtaining the mappings is NOT a constant-time + * operation. Because of the asynchronous nature of the page replacement + * policy, determining the retention ordering requires a traversal of the + * entries. + * + * @return a descending snapshot view of this map + */ + public Map descendingMap() { + return descendingMapWithLimit(Integer.MAX_VALUE); + } + + /** + * Returns an unmodifiable snapshot {@link Map} view of the mappings contained + * in this map. The map's collections return the mappings whose order of + * iteration is the descending order in which its entries are considered + * eligible for retention, from the most-likely to be retained to the + * least-likely. + *

+ * Beware that obtaining the mappings is NOT a constant-time + * operation. Because of the asynchronous nature of the page replacement + * policy, determining the retention ordering requires a traversal of the + * entries. + * + * @param limit the maximum size of the returned map + * @return a descending snapshot view of this map + * @throws IllegalArgumentException if the limit is negative + */ + public Map descendingMapWithLimit(int limit) { + return orderedMap(false, limit); + } + + Map orderedMap(boolean ascending, int limit) { + checkArgument(limit >= 0); + evictionLock.lock(); + try { + drainBuffers(); + + final int initialCapacity = (weigher == Weighers.entrySingleton()) + ? Math.min(limit, (int) weightedSize()) + : 16; + final Map map = new LinkedHashMap(initialCapacity); + final Iterator> iterator = ascending + ? evictionDeque.iterator() + : evictionDeque.descendingIterator(); + while (iterator.hasNext() && (limit > map.size())) { + Node node = iterator.next(); + map.put(node.key, node.getValue()); + } + return unmodifiableMap(map); + } finally { + evictionLock.unlock(); + } + } + + /** The draining status of the buffers. */ + enum DrainStatus { + + /** A drain is not taking place. */ + IDLE { + @Override boolean shouldDrainBuffers(boolean delayable) { + return !delayable; + } + }, + + /** A drain is required due to a pending write modification. */ + REQUIRED { + @Override boolean shouldDrainBuffers(boolean delayable) { + return true; + } + }, + + /** A drain is in progress. */ + PROCESSING { + @Override boolean shouldDrainBuffers(boolean delayable) { + return false; + } + }; + + /** + * Determines whether the buffers should be drained. + * + * @param delayable if a drain should be delayed until required + * @return if a drain should be attempted + */ + abstract boolean shouldDrainBuffers(boolean delayable); + } + + /** A value, its weight, and the entry's status. */ + static final class WeightedValue { + final int weight; + final V value; + + WeightedValue(V value, int weight) { + this.weight = weight; + this.value = value; + } + + boolean contains(Object o) { + return (o == value) || value.equals(o); + } + + /** + * If the entry is available in the hash-table and page replacement policy. + */ + boolean isAlive() { + return weight > 0; + } + + /** + * If the entry was removed from the hash-table and is awaiting removal from + * the page replacement policy. + */ + boolean isRetired() { + return weight < 0; + } + + /** + * If the entry was removed from the hash-table and the page replacement + * policy. + */ + boolean isDead() { + return weight == 0; + } + } + + /** + * A node contains the key, the weighted value, and the linkage pointers on + * the page-replacement algorithm's data structures. + */ + @SuppressWarnings("serial") + static final class Node extends AtomicReference> + implements Linked> { + final K key; + Node prev; + Node next; + + /** Creates a new, unlinked node. */ + Node(K key, WeightedValue weightedValue) { + super(weightedValue); + this.key = key; + } + + @Override + public Node getPrevious() { + return prev; + } + + @Override + public void setPrevious(Node prev) { + this.prev = prev; + } + + @Override + public Node getNext() { + return next; + } + + @Override + public void setNext(Node next) { + this.next = next; + } + + /** Retrieves the value held by the current WeightedValue. */ + V getValue() { + return get().value; + } + } + + /** An adapter to safely externalize the keys. */ + final class KeySet extends AbstractSet { + final ConcurrentLinkedHashMap map = ConcurrentLinkedHashMap.this; + + @Override + public int size() { + return map.size(); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Iterator iterator() { + return new KeyIterator(); + } + + @Override + public boolean contains(Object obj) { + return containsKey(obj); + } + + @Override + public boolean remove(Object obj) { + return (map.remove(obj) != null); + } + + @Override + public Object[] toArray() { + return map.data.keySet().toArray(); + } + + @Override + public T[] toArray(T[] array) { + return map.data.keySet().toArray(array); + } + } + + /** An adapter to safely externalize the key iterator. */ + final class KeyIterator implements Iterator { + final Iterator iterator = data.keySet().iterator(); + K current; + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public K next() { + current = iterator.next(); + return current; + } + + @Override + public void remove() { + checkState(current != null); + ConcurrentLinkedHashMap.this.remove(current); + current = null; + } + } + + /** An adapter to safely externalize the values. */ + final class Values extends AbstractCollection { + + @Override + public int size() { + return ConcurrentLinkedHashMap.this.size(); + } + + @Override + public void clear() { + ConcurrentLinkedHashMap.this.clear(); + } + + @Override + public Iterator iterator() { + return new ValueIterator(); + } + + @Override + public boolean contains(Object o) { + return containsValue(o); + } + } + + /** An adapter to safely externalize the value iterator. */ + final class ValueIterator implements Iterator { + final Iterator> iterator = data.values().iterator(); + Node current; + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public V next() { + current = iterator.next(); + return current.getValue(); + } + + @Override + public void remove() { + checkState(current != null); + ConcurrentLinkedHashMap.this.remove(current.key); + current = null; + } + } + + /** An adapter to safely externalize the entries. */ + final class EntrySet extends AbstractSet> { + final ConcurrentLinkedHashMap map = ConcurrentLinkedHashMap.this; + + @Override + public int size() { + return map.size(); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Iterator> iterator() { + return new EntryIterator(); + } + + @Override + public boolean contains(Object obj) { + if (!(obj instanceof Entry)) { + return false; + } + Entry entry = (Entry) obj; + Node node = map.data.get(entry.getKey()); + return (node != null) && (node.getValue().equals(entry.getValue())); + } + + @Override + public boolean add(Entry entry) { + return (map.putIfAbsent(entry.getKey(), entry.getValue()) == null); + } + + @Override + public boolean remove(Object obj) { + if (!(obj instanceof Entry)) { + return false; + } + Entry entry = (Entry) obj; + return map.remove(entry.getKey(), entry.getValue()); + } + } + + /** An adapter to safely externalize the entry iterator. */ + final class EntryIterator implements Iterator> { + final Iterator> iterator = data.values().iterator(); + Node current; + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Entry next() { + current = iterator.next(); + return new WriteThroughEntry(current); + } + + @Override + public void remove() { + checkState(current != null); + ConcurrentLinkedHashMap.this.remove(current.key); + current = null; + } + } + + /** An entry that allows updates to write through to the map. */ + final class WriteThroughEntry extends SimpleEntry { + static final long serialVersionUID = 1; + + WriteThroughEntry(Node node) { + super(node.key, node.getValue()); + } + + @Override + public V setValue(V value) { + put(getKey(), value); + return super.setValue(value); + } + + Object writeReplace() { + return new SimpleEntry(this); + } + } + + /** A weigher that enforces that the weight falls within a valid range. */ + static final class BoundedEntryWeigher implements EntryWeigher, Serializable { + static final long serialVersionUID = 1; + final EntryWeigher weigher; + + BoundedEntryWeigher(EntryWeigher weigher) { + checkNotNull(weigher); + this.weigher = weigher; + } + + @Override + public int weightOf(K key, V value) { + int weight = weigher.weightOf(key, value); + checkArgument(weight >= 1); + return weight; + } + + Object writeReplace() { + return weigher; + } + } + + /** A queue that discards all additions and is always empty. */ + static final class DiscardingQueue extends AbstractQueue { + @Override public boolean add(Object e) { return true; } + @Override public boolean offer(Object e) { return true; } + @Override public Object poll() { return null; } + @Override public Object peek() { return null; } + @Override public int size() { return 0; } + @Override public Iterator iterator() { return emptyList().iterator(); } + } + + /** A listener that ignores all notifications. */ + enum DiscardingListener implements EvictionListener { + INSTANCE; + + @Override public void onEviction(Object key, Object value) {} + } + + /* ---------------- Serialization Support -------------- */ + + static final long serialVersionUID = 1; + + Object writeReplace() { + return new SerializationProxy(this); + } + + private void readObject(ObjectInputStream stream) throws InvalidObjectException { + throw new InvalidObjectException("Proxy required"); + } + + /** + * A proxy that is serialized instead of the map. The page-replacement + * algorithm's data structures are not serialized so the deserialized + * instance contains only the entries. This is acceptable as caches hold + * transient data that is recomputable and serialization would tend to be + * used as a fast warm-up process. + */ + static final class SerializationProxy implements Serializable { + final EntryWeigher weigher; + final EvictionListener listener; + final int concurrencyLevel; + final Map data; + final long capacity; + + SerializationProxy(ConcurrentLinkedHashMap map) { + concurrencyLevel = map.concurrencyLevel; + data = new HashMap(map); + capacity = map.capacity.get(); + listener = map.listener; + weigher = map.weigher; + } + + Object readResolve() { + ConcurrentLinkedHashMap map = new Builder() + .concurrencyLevel(concurrencyLevel) + .maximumWeightedCapacity(capacity) + .listener(listener) + .weigher(weigher) + .build(); + map.putAll(data); + return map; + } + + static final long serialVersionUID = 1; + } + + /* ---------------- Builder -------------- */ + + /** + * A builder that creates {@link ConcurrentLinkedHashMap} instances. It + * provides a flexible approach for constructing customized instances with + * a named parameter syntax. It can be used in the following manner: + * 
{@code
+   * ConcurrentMap> graph = new Builder>()
+   *     .maximumWeightedCapacity(5000)
+   *     .weigher(Weighers.set())
+   *     .build();
+   * }
+ */ + public static final class Builder { + static final int DEFAULT_CONCURRENCY_LEVEL = 16; + static final int DEFAULT_INITIAL_CAPACITY = 16; + + EvictionListener listener; + EntryWeigher weigher; + + int concurrencyLevel; + int initialCapacity; + long capacity; + + @SuppressWarnings("unchecked") + public Builder() { + capacity = -1; + weigher = Weighers.entrySingleton(); + initialCapacity = DEFAULT_INITIAL_CAPACITY; + concurrencyLevel = DEFAULT_CONCURRENCY_LEVEL; + listener = (EvictionListener) DiscardingListener.INSTANCE; + } + + /** + * Specifies the initial capacity of the hash table (default 16). + * This is the number of key-value pairs that the hash table can hold + * before a resize operation is required. + * + * @param initialCapacity the initial capacity used to size the hash table + * to accommodate this many entries. + * @throws IllegalArgumentException if the initialCapacity is negative + */ + public Builder initialCapacity(int initialCapacity) { + checkArgument(initialCapacity >= 0); + this.initialCapacity = initialCapacity; + return this; + } + + /** + * Specifies the maximum weighted capacity to coerce the map to and may + * exceed it temporarily. + * + * @param capacity the weighted threshold to bound the map by + * @throws IllegalArgumentException if the maximumWeightedCapacity is + * negative + */ + public Builder maximumWeightedCapacity(long capacity) { + checkArgument(capacity >= 0); + this.capacity = capacity; + return this; + } + + /** + * Specifies the estimated number of concurrently updating threads. The + * implementation performs internal sizing to try to accommodate this many + * threads (default 16). + * + * @param concurrencyLevel the estimated number of concurrently updating + * threads + * @throws IllegalArgumentException if the concurrencyLevel is less than or + * equal to zero + */ + public Builder concurrencyLevel(int concurrencyLevel) { + checkArgument(concurrencyLevel > 0); + this.concurrencyLevel = concurrencyLevel; + return this; + } + + /** + * Specifies an optional listener that is registered for notification when + * an entry is evicted. + * + * @param listener the object to forward evicted entries to + * @throws NullPointerException if the listener is null + */ + public Builder listener(EvictionListener listener) { + checkNotNull(listener); + this.listener = listener; + return this; + } + + /** + * Specifies an algorithm to determine how many the units of capacity a + * value consumes. The default algorithm bounds the map by the number of + * key-value pairs by giving each entry a weight of 1. + * + * @param weigher the algorithm to determine a value's weight + * @throws NullPointerException if the weigher is null + */ + public Builder weigher(Weigher weigher) { + this.weigher = (weigher == Weighers.singleton()) + ? Weighers.entrySingleton() + : new BoundedEntryWeigher(Weighers.asEntryWeigher(weigher)); + return this; + } + + /** + * Specifies an algorithm to determine how many the units of capacity an + * entry consumes. The default algorithm bounds the map by the number of + * key-value pairs by giving each entry a weight of 1. + * + * @param weigher the algorithm to determine a entry's weight + * @throws NullPointerException if the weigher is null + */ + public Builder weigher(EntryWeigher weigher) { + this.weigher = (weigher == Weighers.entrySingleton()) + ? Weighers.entrySingleton() + : new BoundedEntryWeigher(weigher); + return this; + } + + /** + * Creates a new {@link ConcurrentLinkedHashMap} instance. + * + * @throws IllegalStateException if the maximum weighted capacity was + * not set + */ + public ConcurrentLinkedHashMap build() { + checkState(capacity >= 0); + return new ConcurrentLinkedHashMap(this); + } + } +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java new file mode 100644 index 000000000..d07423c2e --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java @@ -0,0 +1,37 @@ +/* + * Copyright 2012 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +/** + * A class that can determine the weight of an entry. The total weight threshold + * is used to determine when an eviction is required. + * + * @author ben.manes@gmail.com (Ben Manes) + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +public interface EntryWeigher { + + /** + * Measures an entry's weight to determine how many units of capacity that + * the key and value consumes. An entry must consume a minimum of one unit. + * + * @param key the key to weigh + * @param value the value to weigh + * @return the entry's weight + */ + int weightOf(K key, V value); +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java new file mode 100644 index 000000000..6b3ac196d --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java @@ -0,0 +1,45 @@ +/* + * Copyright 2010 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +/** + * A listener registered for notification when an entry is evicted. An instance + * may be called concurrently by multiple threads to process entries. An + * implementation should avoid performing blocking calls or synchronizing on + * shared resources. + *

+ * The listener is invoked by {@link ConcurrentLinkedHashMap} on a caller's + * thread and will not block other threads from operating on the map. An + * implementation should be aware that the caller's thread will not expect + * long execution times or failures as a side effect of the listener being + * notified. Execution safety and a fast turn around time can be achieved by + * performing the operation asynchronously, such as by submitting a task to an + * {@link java.util.concurrent.ExecutorService}. + * + * @author ben.manes@gmail.com (Ben Manes) + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +public interface EvictionListener { + + /** + * A call-back notification that the entry was evicted. + * + * @param key the entry's key + * @param value the entry's value + */ + void onEviction(K key, V value); +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE b/src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java new file mode 100644 index 000000000..0354a69f6 --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java @@ -0,0 +1,460 @@ +/* + * Copyright 2011 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +import java.util.AbstractCollection; +import java.util.Collection; +import java.util.Deque; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Linked list implementation of the {@link Deque} interface where the link + * pointers are tightly integrated with the element. Linked deques have no + * capacity restrictions; they grow as necessary to support usage. They are not + * thread-safe; in the absence of external synchronization, they do not support + * concurrent access by multiple threads. Null elements are prohibited. + *

+ * Most LinkedDeque operations run in constant time by assuming that + * the {@link Linked} parameter is associated with the deque instance. Any usage + * that violates this assumption will result in non-deterministic behavior. + *

+ * The iterators returned by this class are not fail-fast: If + * the deque is modified at any time after the iterator is created, the iterator + * will be in an unknown state. Thus, in the face of concurrent modification, + * the iterator risks arbitrary, non-deterministic behavior at an undetermined + * time in the future. + * + * @author ben.manes@gmail.com (Ben Manes) + * @param the type of elements held in this collection + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +final class LinkedDeque> extends AbstractCollection implements Deque { + + // This class provides a doubly-linked list that is optimized for the virtual + // machine. The first and last elements are manipulated instead of a slightly + // more convenient sentinel element to avoid the insertion of null checks with + // NullPointerException throws in the byte code. The links to a removed + // element are cleared to help a generational garbage collector if the + // discarded elements inhabit more than one generation. + + /** + * Pointer to first node. + * Invariant: (first == null && last == null) || + * (first.prev == null) + */ + E first; + + /** + * Pointer to last node. + * Invariant: (first == null && last == null) || + * (last.next == null) + */ + E last; + + /** + * Links the element to the front of the deque so that it becomes the first + * element. + * + * @param e the unlinked element + */ + void linkFirst(final E e) { + final E f = first; + first = e; + + if (f == null) { + last = e; + } else { + f.setPrevious(e); + e.setNext(f); + } + } + + /** + * Links the element to the back of the deque so that it becomes the last + * element. + * + * @param e the unlinked element + */ + void linkLast(final E e) { + final E l = last; + last = e; + + if (l == null) { + first = e; + } else { + l.setNext(e); + e.setPrevious(l); + } + } + + /** Unlinks the non-null first element. */ + E unlinkFirst() { + final E f = first; + final E next = f.getNext(); + f.setNext(null); + + first = next; + if (next == null) { + last = null; + } else { + next.setPrevious(null); + } + return f; + } + + /** Unlinks the non-null last element. */ + E unlinkLast() { + final E l = last; + final E prev = l.getPrevious(); + l.setPrevious(null); + last = prev; + if (prev == null) { + first = null; + } else { + prev.setNext(null); + } + return l; + } + + /** Unlinks the non-null element. */ + void unlink(E e) { + final E prev = e.getPrevious(); + final E next = e.getNext(); + + if (prev == null) { + first = next; + } else { + prev.setNext(next); + e.setPrevious(null); + } + + if (next == null) { + last = prev; + } else { + next.setPrevious(prev); + e.setNext(null); + } + } + + @Override + public boolean isEmpty() { + return (first == null); + } + + void checkNotEmpty() { + if (isEmpty()) { + throw new NoSuchElementException(); + } + } + + /** + * {@inheritDoc} + *

+ * Beware that, unlike in most collections, this method is NOT a + * constant-time operation. + */ + @Override + public int size() { + int size = 0; + for (E e = first; e != null; e = e.getNext()) { + size++; + } + return size; + } + + @Override + public void clear() { + for (E e = first; e != null;) { + E next = e.getNext(); + e.setPrevious(null); + e.setNext(null); + e = next; + } + first = last = null; + } + + @Override + public boolean contains(Object o) { + return (o instanceof Linked) && contains((Linked) o); + } + + // A fast-path containment check + boolean contains(Linked e) { + return (e.getPrevious() != null) + || (e.getNext() != null) + || (e == first); + } + + /** + * Moves the element to the front of the deque so that it becomes the first + * element. + * + * @param e the linked element + */ + public void moveToFront(E e) { + if (e != first) { + unlink(e); + linkFirst(e); + } + } + + /** + * Moves the element to the back of the deque so that it becomes the last + * element. + * + * @param e the linked element + */ + public void moveToBack(E e) { + if (e != last) { + unlink(e); + linkLast(e); + } + } + + @Override + public E peek() { + return peekFirst(); + } + + @Override + public E peekFirst() { + return first; + } + + @Override + public E peekLast() { + return last; + } + + @Override + public E getFirst() { + checkNotEmpty(); + return peekFirst(); + } + + @Override + public E getLast() { + checkNotEmpty(); + return peekLast(); + } + + @Override + public E element() { + return getFirst(); + } + + @Override + public boolean offer(E e) { + return offerLast(e); + } + + @Override + public boolean offerFirst(E e) { + if (contains(e)) { + return false; + } + linkFirst(e); + return true; + } + + @Override + public boolean offerLast(E e) { + if (contains(e)) { + return false; + } + linkLast(e); + return true; + } + + @Override + public boolean add(E e) { + return offerLast(e); + } + + + @Override + public void addFirst(E e) { + if (!offerFirst(e)) { + throw new IllegalArgumentException(); + } + } + + @Override + public void addLast(E e) { + if (!offerLast(e)) { + throw new IllegalArgumentException(); + } + } + + @Override + public E poll() { + return pollFirst(); + } + + @Override + public E pollFirst() { + return isEmpty() ? null : unlinkFirst(); + } + + @Override + public E pollLast() { + return isEmpty() ? null : unlinkLast(); + } + + @Override + public E remove() { + return removeFirst(); + } + + @Override + @SuppressWarnings("unchecked") + public boolean remove(Object o) { + return (o instanceof Linked) && remove((E) o); + } + + // A fast-path removal + boolean remove(E e) { + if (contains(e)) { + unlink(e); + return true; + } + return false; + } + + @Override + public E removeFirst() { + checkNotEmpty(); + return pollFirst(); + } + + @Override + public boolean removeFirstOccurrence(Object o) { + return remove(o); + } + + @Override + public E removeLast() { + checkNotEmpty(); + return pollLast(); + } + + @Override + public boolean removeLastOccurrence(Object o) { + return remove(o); + } + + @Override + public boolean removeAll(Collection c) { + boolean modified = false; + for (Object o : c) { + modified |= remove(o); + } + return modified; + } + + @Override + public void push(E e) { + addFirst(e); + } + + @Override + public E pop() { + return removeFirst(); + } + + @Override + public Iterator iterator() { + return new AbstractLinkedIterator(first) { + @Override E computeNext() { + return cursor.getNext(); + } + }; + } + + @Override + public Iterator descendingIterator() { + return new AbstractLinkedIterator(last) { + @Override E computeNext() { + return cursor.getPrevious(); + } + }; + } + + abstract class AbstractLinkedIterator implements Iterator { + E cursor; + + /** + * Creates an iterator that can can traverse the deque. + * + * @param start the initial element to begin traversal from + */ + AbstractLinkedIterator(E start) { + cursor = start; + } + + @Override + public boolean hasNext() { + return (cursor != null); + } + + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + E e = cursor; + cursor = computeNext(); + return e; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + /** + * Retrieves the next element to traverse to or null if there are + * no more elements. + */ + abstract E computeNext(); + } +} + +/** + * An element that is linked on the {@link Deque}. + */ +interface Linked> { + + /** + * Retrieves the previous element or null if either the element is + * unlinked or the first element on the deque. + */ + T getPrevious(); + + /** Sets the previous element or null if there is no link. */ + void setPrevious(T prev); + + /** + * Retrieves the next element or null if either the element is + * unlinked or the last element on the deque. + */ + T getNext(); + + /** Sets the next element or null if there is no link. */ + void setNext(T next); +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE b/src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE new file mode 100644 index 000000000..e1cedae49 --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE @@ -0,0 +1,7 @@ +ConcurrentLinkedHashMap +Copyright 2008, Ben Manes +Copyright 2010, Google Inc. + +Some alternate data structures provided by JSR-166e +from http://gee.cs.oswego.edu/dl/concurrency-interest/. +Written by Doug Lea and released as Public Domain. diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java new file mode 100644 index 000000000..2fef7f0e7 --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java @@ -0,0 +1,36 @@ +/* + * Copyright 2010 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +/** + * A class that can determine the weight of a value. The total weight threshold + * is used to determine when an eviction is required. + * + * @author ben.manes@gmail.com (Ben Manes) + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +public interface Weigher { + + /** + * Measures an object's weight to determine how many units of capacity that + * the value consumes. A value must consume a minimum of one unit. + * + * @param value the object to weigh + * @return the object's weight + */ + int weightOf(V value); +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java new file mode 100644 index 000000000..c3c11a152 --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java @@ -0,0 +1,282 @@ +/* + * Copyright 2010 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.googlecode.concurrentlinkedhashmap; + +import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.checkNotNull; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A common set of {@link Weigher} and {@link EntryWeigher} implementations. + * + * @author ben.manes@gmail.com (Ben Manes) + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +public final class Weighers { + + private Weighers() { + throw new AssertionError(); + } + + /** + * A entry weigher backed by the specified weigher. The weight of the value + * determines the weight of the entry. + * + * @param weigher the weigher to be "wrapped" in a entry weigher. + * @return A entry weigher view of the specified weigher. + */ + public static EntryWeigher asEntryWeigher( + final Weigher weigher) { + return (weigher == singleton()) + ? Weighers.entrySingleton() + : new EntryWeigherView(weigher); + } + + /** + * A weigher where an entry has a weight of 1. A map bounded with + * this weigher will evict when the number of key-value pairs exceeds the + * capacity. + * + * @return A weigher where a value takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static EntryWeigher entrySingleton() { + return (EntryWeigher) SingletonEntryWeigher.INSTANCE; + } + + /** + * A weigher where a value has a weight of 1. A map bounded with + * this weigher will evict when the number of key-value pairs exceeds the + * capacity. + * + * @return A weigher where a value takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher singleton() { + return (Weigher) SingletonWeigher.INSTANCE; + } + + /** + * A weigher where the value is a byte array and its weight is the number of + * bytes. A map bounded with this weigher will evict when the number of bytes + * exceeds the capacity rather than the number of key-value pairs in the map. + * This allows for restricting the capacity based on the memory-consumption + * and is primarily for usage by dedicated caching servers that hold the + * serialized data. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each byte takes one unit of capacity. + */ + public static Weigher byteArray() { + return ByteArrayWeigher.INSTANCE; + } + + /** + * A weigher where the value is a {@link Iterable} and its weight is the + * number of elements. This weigher only should be used when the alternative + * {@link #collection()} weigher cannot be, as evaluation takes O(n) time. A + * map bounded with this weigher will evict when the total number of elements + * exceeds the capacity rather than the number of key-value pairs in the map. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each element takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher> iterable() { + return (Weigher>) (Weigher) IterableWeigher.INSTANCE; + } + + /** + * A weigher where the value is a {@link Collection} and its weight is the + * number of elements. A map bounded with this weigher will evict when the + * total number of elements exceeds the capacity rather than the number of + * key-value pairs in the map. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each element takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher> collection() { + return (Weigher>) (Weigher) CollectionWeigher.INSTANCE; + } + + /** + * A weigher where the value is a {@link List} and its weight is the number + * of elements. A map bounded with this weigher will evict when the total + * number of elements exceeds the capacity rather than the number of + * key-value pairs in the map. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each element takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher> list() { + return (Weigher>) (Weigher) ListWeigher.INSTANCE; + } + + /** + * A weigher where the value is a {@link Set} and its weight is the number + * of elements. A map bounded with this weigher will evict when the total + * number of elements exceeds the capacity rather than the number of + * key-value pairs in the map. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each element takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher> set() { + return (Weigher>) (Weigher) SetWeigher.INSTANCE; + } + + /** + * A weigher where the value is a {@link Map} and its weight is the number of + * entries. A map bounded with this weigher will evict when the total number of + * entries across all values exceeds the capacity rather than the number of + * key-value pairs in the map. + *

+ * A value with a weight of 0 will be rejected by the map. If a value + * with this weight can occur then the caller should eagerly evaluate the + * value and treat it as a removal operation. Alternatively, a custom weigher + * may be specified on the map to assign an empty value a positive weight. + * + * @return A weigher where each entry takes one unit of capacity. + */ + @SuppressWarnings({"cast", "unchecked"}) + public static Weigher> map() { + return (Weigher>) (Weigher) MapWeigher.INSTANCE; + } + + static final class EntryWeigherView implements EntryWeigher, Serializable { + static final long serialVersionUID = 1; + final Weigher weigher; + + EntryWeigherView(Weigher weigher) { + checkNotNull(weigher); + this.weigher = weigher; + } + + @Override + public int weightOf(K key, V value) { + return weigher.weightOf(value); + } + } + + enum SingletonEntryWeigher implements EntryWeigher { + INSTANCE; + + @Override + public int weightOf(Object key, Object value) { + return 1; + } + } + + enum SingletonWeigher implements Weigher { + INSTANCE; + + @Override + public int weightOf(Object value) { + return 1; + } + } + + enum ByteArrayWeigher implements Weigher { + INSTANCE; + + @Override + public int weightOf(byte[] value) { + return value.length; + } + } + + enum IterableWeigher implements Weigher> { + INSTANCE; + + @Override + public int weightOf(Iterable values) { + if (values instanceof Collection) { + return ((Collection) values).size(); + } + int size = 0; + for (Iterator i = values.iterator(); i.hasNext();) { + i.next(); + size++; + } + return size; + } + } + + enum CollectionWeigher implements Weigher> { + INSTANCE; + + @Override + public int weightOf(Collection values) { + return values.size(); + } + } + + enum ListWeigher implements Weigher> { + INSTANCE; + + @Override + public int weightOf(List values) { + return values.size(); + } + } + + enum SetWeigher implements Weigher> { + INSTANCE; + + @Override + public int weightOf(Set values) { + return values.size(); + } + } + + enum MapWeigher implements Weigher> { + INSTANCE; + + @Override + public int weightOf(Map values) { + return values.size(); + } + } +} diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java b/src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java new file mode 100644 index 000000000..57bab113b --- /dev/null +++ b/src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java @@ -0,0 +1,41 @@ +/* + * Copyright 2011 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains an implementation of a bounded + * {@link java.util.concurrent.ConcurrentMap} data structure. + *

+ * {@link com.googlecode.concurrentlinkedhashmap.Weigher} is a simple interface + * for determining how many units of capacity an entry consumes. Depending on + * which concrete Weigher class is used, an entry may consume a different amount + * of space within the cache. The + * {@link com.googlecode.concurrentlinkedhashmap.Weighers} class provides + * utility methods for obtaining the most common kinds of implementations. + *

+ * {@link com.googlecode.concurrentlinkedhashmap.EvictionListener} provides the + * ability to be notified when an entry is evicted from the map. An eviction + * occurs when the entry was automatically removed due to the map exceeding a + * capacity threshold. It is not called when an entry was explicitly removed. + *

+ * The {@link com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap} + * class supplies an efficient, scalable, thread-safe, bounded map. As with the + * Java Collections Framework the "Concurrent" prefix is used to + * indicate that the map is not governed by a single exclusion lock. + * + * @see + * http://code.google.com/p/concurrentlinkedhashmap/ + */ +package com.googlecode.concurrentlinkedhashmap; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java similarity index 99% rename from src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java rename to src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java index 0ae2db889..3b2fc4f78 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParsedSQLCacheItem.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java @@ -25,3 +25,4 @@ final class ParsedSQLCacheItem { this.bReturnValueSyntax = bReturnValueSyntax; } } + diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index b64fb331e..30237a67a 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -56,10 +56,9 @@ import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;; +import com.googlecode.concurrentlinkedhashmap.EvictionListener; /** * SQLServerConnection implements a JDBC connection to SQL Server. SQLServerConnections support JDBC connection pooling and may be either physical @@ -127,6 +126,7 @@ final class PreparedStatementCacheItem { int handle; boolean hasExecutedSpExecuteSql; boolean handleIsDirectSql; + boolean evictedFromCache; SQLServerConnection connection; private AtomicInteger refCount = new AtomicInteger(1); SQLServerParameterMetaData parameterMetadata; @@ -181,7 +181,7 @@ void decrementHandleRefCount() { private int statementPoolingCacheSize = 10; /** Cache of prepared statement handles */ - private Cache preparedStatementCache; + private ConcurrentLinkedHashMap preparedStatementCache; SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; @@ -2783,7 +2783,7 @@ public void close() throws SQLServerException { // Invalidate statement cache. if(null != this.preparedStatementCache) - this.preparedStatementCache.invalidateAll(); + this.preparedStatementCache.clear(); // Clean-up queue etc. related to batching of prepared statement discard actions (sp_unprepare). cleanupPreparedStatementDiscardActions(); @@ -5590,56 +5590,56 @@ public void setStatementPoolingCacheSize(int value) { } /** Get prepared statement cache entry if exists */ - final PreparedStatementCacheItem getCachedPreparedStatementMetadata(String sql) { + final PreparedStatementCacheItem getCachedPreparedStatementMetadata(SQLServerPreparedStatement.Sha1HashKey key) { if(null == this.preparedStatementCache) return null; - PreparedStatementCacheItem cacheItem = this.preparedStatementCache.getIfPresent(sql); - - return cacheItem; + if(null == key) + return null; + else + return this.preparedStatementCache.get(key); } // Handle closing handles when removed from cache. - RemovalListener preparedStatementHandleCacheRemovalListener = new RemovalListener() { - public void onRemoval(RemovalNotification removal) { - PreparedStatementCacheItem cacheItem = removal.getValue(); - // Only discard if not referenced. - if(null != cacheItem && cacheItem.discardIfHandleNotReferenced()) { - if(cacheItem.hasHandle()) { + final class PreparedStatementHandleEvictionListener + implements EvictionListener { + public void onEviction(SQLServerPreparedStatement.Sha1HashKey key, PreparedStatementCacheItem cacheItem) { + if(null != cacheItem) { + cacheItem.evictedFromCache = true; // Mark as evicted from cache. + + // Only discard if not referenced. + if(cacheItem.hasHandle() && cacheItem.discardIfHandleNotReferenced()) { cacheItem.connection.enqueuePreparedStatementDiscardItem(cacheItem.handle, cacheItem.handleIsDirectSql); cacheItem.connection.handlePreparedStatementDiscardActions(false); } } - else if(null != cacheItem) - // Put back in cache. - cacheItem.connection.cachePreparedStatementMetadata(removal.getKey(), cacheItem); } - }; - + } + /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementExecuteSqlUse(String sql) { + final void cachePreparedStatementExecuteSqlUse(SQLServerPreparedStatement.Sha1HashKey key) { // Caching turned off? if(0 >= this.getStatementPoolingCacheSize()) return; - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); if(null != cacheItem) cacheItem.hasExecutedSpExecuteSql = true; else { cacheItem = new PreparedStatementCacheItem(0, false, true, null, this); - this.cachePreparedStatementMetadata(sql, cacheItem); + this.cachePreparedStatementMetadata(key, cacheItem); } } /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementHandle(String sql, int handle, boolean directSql, SQLServerPreparedStatement statement) { + final void cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey key, int handle, boolean directSql, SQLServerPreparedStatement statement) { // Caching turned off? if(0 >= this.getStatementPoolingCacheSize()) return; - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); if(null != cacheItem) { cacheItem.handle = handle; @@ -5648,36 +5648,38 @@ final void cachePreparedStatementHandle(String sql, int handle, boolean directSq else { cacheItem = new PreparedStatementCacheItem(handle, directSql, false, null, this); - this.cachePreparedStatementMetadata(sql, cacheItem); + this.cachePreparedStatementMetadata(key, cacheItem); } } /** Add cache entry for prepared statement metadata*/ - final void cacheParameterMetadata(String sql, SQLServerParameterMetaData metadata, SQLServerPreparedStatement statement) { + final void cacheParameterMetadata(SQLServerPreparedStatement.Sha1HashKey key, SQLServerParameterMetaData metadata, SQLServerPreparedStatement statement) { // Caching turned off? if(0 >= this.getStatementPoolingCacheSize()) return; - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(sql); + PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); if(null != cacheItem) cacheItem.parameterMetadata = metadata; else - this.cachePreparedStatementMetadata(sql, new PreparedStatementCacheItem(0, false, false, metadata, this)); + this.cachePreparedStatementMetadata(key, new PreparedStatementCacheItem(0, false, false, metadata, this)); } /** Add cache entry for prepared statement metadata*/ - private void cachePreparedStatementMetadata(String sql, PreparedStatementCacheItem cacheItem) { + private void cachePreparedStatementMetadata(SQLServerPreparedStatement.Sha1HashKey key, PreparedStatementCacheItem cacheItem) { // Caching turned off? if(0 >= this.getStatementPoolingCacheSize() || null == cacheItem) return; if(null == this.preparedStatementCache) { - preparedStatementCache = CacheBuilder.newBuilder() - .maximumSize(this.getStatementPoolingCacheSize()) - .build(); + this.preparedStatementCache = new Builder() + .maximumWeightedCapacity(this.getStatementPoolingCacheSize()) + .listener(new PreparedStatementHandleEvictionListener()) + .build(); } - this.preparedStatementCache.put(sql, cacheItem); + if(null != key) + this.preparedStatementCache.put(key, cacheItem); } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java index c0e26d646..b779a735e 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java @@ -551,7 +551,9 @@ String parseThreePartNames(String threeName) throws SQLServerException { } private void checkClosed() throws SQLServerException { - stmtParent.checkClosed(); + // stmtParent does not seem to be re-used, should just verify connection is not closed. + // stmtParent.checkClosed(); + con.checkClosed(); } /** diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index d1d42cbe2..a9847b5cb 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -28,8 +28,8 @@ import java.util.Vector; import java.util.logging.Level; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.Cache; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;; /** * SQLServerPreparedStatement provides JDBC prepared statement functionality. SQLServerPreparedStatement provides methods for the user to supply @@ -55,12 +55,12 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS final int nBatchStatementDelimiter = BATCH_STATEMENT_DELIMITER_TDS_72; /** the user's prepared sql syntax */ - private String sqlCommand; + //private String sqlCommand; /** The prepared type definitions */ private String preparedTypeDefinitions; - /** Processed SQL statement text that will be executed, may not be same as what user initially passed (which is available in sqlCommand) */ + /** Processed SQL statement text that will be executed, may not be same as what user initially passed. */ final String userSQL; /** SQL statement with expanded parameter tokens */ @@ -126,7 +126,7 @@ private void setPrepStmtHandle(int handle, String sql, boolean cache) { prepStmtHandle = handle; if(cache) - connection.cachePreparedStatementHandle(sql, handle, executedSqlDirectly, this); + connection.cachePreparedStatementHandle(cacheKey, handle, executedSqlDirectly, this); } /** Resets the server handle for this prepared statement to no handle. @@ -147,21 +147,51 @@ private void resetPrepStmtHandle() { /** Size of the parsed SQL-text metadata cache */ static final private int parsedSQLCacheSize = 100; + static class Sha1HashKey { + private byte[] bytes; + + Sha1HashKey(String s) { + bytes = getSha1Digest().digest(s.getBytes()); + } + + public boolean equals(Object obj) { + return java.util.Arrays.equals(bytes, ((Sha1HashKey)obj).bytes); + } + + public int hashCode() { + return java.util.Arrays.hashCode(bytes); + } + + private java.security.MessageDigest getSha1Digest() { + try { + return java.security.MessageDigest.getInstance("SHA-1"); + } + catch (final java.security.NoSuchAlgorithmException e) { + // This is not theoretically possible, but we're forced to catch it anyway + throw new RuntimeException(e); + } + } + } + /** Cache of prepared statement meta data */ - static private Cache parsedSQLCache; + static private ConcurrentLinkedHashMap parsedSQLCache; + static { - parsedSQLCache = CacheBuilder.newBuilder() - .maximumSize(parsedSQLCacheSize) - .build(); + parsedSQLCache = new Builder() + .maximumWeightedCapacity(parsedSQLCacheSize) + .build(); } /** Get prepared statement cache entry if exists */ - static ParsedSQLCacheItem getCachedParsedSQLMetadata(String initialSql) { - return parsedSQLCache.getIfPresent(initialSql); + static ParsedSQLCacheItem getCachedParsedSQLMetadata(Sha1HashKey key) { + if(null == key) + return null; + else + return parsedSQLCache.get(key); } /** Add cache entry for prepared statement metadata*/ - static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql) throws SQLServerException { + static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql, Sha1HashKey key) throws SQLServerException { JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); @@ -172,7 +202,8 @@ static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql) throws SQL // Cache this entry. ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); - parsedSQLCache.put(initialSql, cacheItem); + if(null != initialSql) + parsedSQLCache.put(key, cacheItem); return cacheItem; } @@ -182,6 +213,9 @@ String getClassNameInternal() { return "SQLServerPreparedStatement"; } + /** Key used to lookup this statement in caches. */ + private Sha1HashKey cacheKey; + /** * Create a new prepaed statement. * @@ -205,17 +239,16 @@ String getClassNameInternal() { SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException { super(conn, nRSType, nRSConcur, stmtColEncSetting); stmtPoolable = true; - sqlCommand = sql; - // Save original SQL statement. - sqlCommand = sql; - + // Create a cache key for this statement. + cacheKey = new Sha1HashKey(sql); + // Check for cached SQL metadata. - ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(sql); + ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(cacheKey); // No cached meta data found, parse. if(null == cacheItem) - cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql); + cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql, cacheKey); // Retrieve from cache item. procedureName = cacheItem.procedureName; @@ -250,12 +283,20 @@ private void closePreparedHandle() { if(1 < connection.getServerPreparedStatementDiscardThreshold()) { // Handle unprepare actions through batching @ connection level. // Use this only if statement caching is off, otherwise this will be called by statement cache invalidation. - if(!this.connection.isStatementPoolingEnabled()) { + if(null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) + cachedPreparedStatementHandle.decrementHandleRefCount(); + + if( + !this.connection.isStatementPoolingEnabled() // No caching + || ( + null != cachedPreparedStatementHandle // Cache ref. exists + && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache + && cachedPreparedStatementHandle.discardIfHandleNotReferenced() // Not used by any other statements. + ) + ) { connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); connection.handlePreparedStatementDiscardActions(false); } - else if(null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) - cachedPreparedStatementHandle.decrementHandleRefCount(); } else { // Non batched behavior (same as pre batch impl.) @@ -583,7 +624,7 @@ else if (EXECUTE_UPDATE == executeMethod && null != resultSet) { private void handleUsingCachedStmtHandle() { if(!hasPreparedStatementHandle()) { // Check for cached handle. - SQLServerConnection.PreparedStatementCacheItem cachedHandle = this.connection.getCachedPreparedStatementMetadata(userSQL); + SQLServerConnection.PreparedStatementCacheItem cachedHandle = this.connection.getCachedPreparedStatementMetadata(cacheKey); // If handle was found then re-use. if(null != cachedHandle && (cachedHandle.hasHandle() || cachedHandle.hasExecutedSpExecuteSql)) { @@ -961,7 +1002,7 @@ private boolean doPrepExec(TDSWriter tdsWriter, isExecutedAtLeastOnce = true; // Enable re-use if caching is on by moving to sp_prepexec on next call even from separate instance. - connection.cachePreparedStatementExecuteSqlUse(userSQL); + connection.cachePreparedStatementExecuteSqlUse(cacheKey); } // Second execution, use prepared statements since we seem to be re-using it. else if(needsPrepare) @@ -1008,11 +1049,12 @@ else if (resultSet != null) { * @return the result set containing the meta data */ /* L0 */ private ResultSet buildExecuteMetaData() throws SQLServerException { - String fmtSQL = sqlCommand; + String fmtSQL = userSQL; + /* if (fmtSQL.indexOf(LEFT_CURLY_BRACKET) >= 0) { fmtSQL = userSQL; } - + */ ResultSet emptyResultSet = null; try { fmtSQL = replaceMarkerWithNull(fmtSQL); @@ -2937,7 +2979,7 @@ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws SQLServerConnection.PreparedStatementCacheItem cacheItem = null; if( !forceRefresh - && null != (cacheItem = connection.getCachedPreparedStatementMetadata(userSQL)) + && null != (cacheItem = connection.getCachedPreparedStatementMetadata(cacheKey)) && cacheItem.hasParameterMetadata() ) { return cacheItem.parameterMetadata; @@ -2947,7 +2989,7 @@ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws checkClosed(); SQLServerParameterMetaData pmd = new SQLServerParameterMetaData(this, userSQL); - connection.cacheParameterMetadata(userSQL, pmd, this); + connection.cacheParameterMetadata(cacheKey, pmd, this); loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index 63776dd64..ec43e4106 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -762,12 +762,14 @@ final void processResponse(TDSReader tdsReader) throws SQLServerException { private String ensureSQLSyntax(String sql) throws SQLServerException { if (sql.indexOf(LEFT_CURLY_BRACKET) >= 0) { + SQLServerPreparedStatement.Sha1HashKey cacheKey = new SQLServerPreparedStatement.Sha1HashKey(sql); + // Check for cached SQL metadata. - ParsedSQLCacheItem cacheItem = SQLServerPreparedStatement.getCachedParsedSQLMetadata(sql); + ParsedSQLCacheItem cacheItem = SQLServerPreparedStatement.getCachedParsedSQLMetadata(cacheKey); // No cached SQL-text meta datafound, parse. if(null == cacheItem) - cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql); + cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql, cacheKey); // Retrieve from cache item. procedureName = cacheItem.procedureName; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 3eab507ab..b0561547d 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -154,27 +154,26 @@ public void testStatementPooling() throws SQLException { // Execute statement first, should create cache entry WITHOUT handle (since sp_executesql was used). try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { pstmt.execute(); // sp_executesql + pstmt.getMoreResults(); // Make sure handle is updated. assertSame(0, pstmt.getPreparedStatementHandle()); } // Execute statement again, should now create handle. - try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { - pstmt.execute(); // sp_prepexec - } - - // Execute statement again and save handle, should now have used handle from cache. int handle = 0; try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { - pstmt.execute(); // sp_execute + pstmt.execute(); // sp_prepexec + + pstmt.getMoreResults(); // Make sure handle is updated. handle = pstmt.getPreparedStatementHandle(); - assertNotSame(0, pstmt.getPreparedStatementHandle()); + assertNotSame(0, handle); } // Execute statement again and verify same handle was used. try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { pstmt.execute(); // sp_execute + pstmt.getMoreResults(); // Make sure handle is updated. assertNotSame(0, pstmt.getPreparedStatementHandle()); assertSame(handle, pstmt.getPreparedStatementHandle()); @@ -183,6 +182,7 @@ public void testStatementPooling() throws SQLException { // Execute new statement with different SQL text and verify it does NOT get same handle (should now fall back to using sp_executesql). try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + ";")) { pstmt.execute(); // sp_executesql + pstmt.getMoreResults(); // Make sure handle is updated. assertSame(0, pstmt.getPreparedStatementHandle()); assertNotSame(handle, pstmt.getPreparedStatementHandle()); From f89d7409e907a0d0d77b51ef8fac1822d5f85bb7 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Sat, 6 May 2017 13:03:06 -0500 Subject: [PATCH 12/34] Clean-up --- .../sqlserver/jdbc/ParsedSQLCacheItem.java | 8 ++-- .../sqlserver/jdbc/SQLServerConnection.java | 16 ++++---- .../jdbc/SQLServerPreparedStatement.java | 37 ++++++++----------- .../sqlserver/jdbc/SQLServerStatement.java | 2 +- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java index 3b2fc4f78..19c34ebec 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java @@ -9,17 +9,17 @@ package com.microsoft.sqlserver.jdbc; /** - * Used to keep track of parsed SQL text and its properties for prepared statements. + * Used for caching of meta data from parsed SQL text. */ final class ParsedSQLCacheItem { /** The SQL text AFTER processing. */ - String preparedSQLText; + String processedSQL; int parameterCount; String procedureName; boolean bReturnValueSyntax; - ParsedSQLCacheItem(String preparedSQLText, int parameterCount, String procedureName, boolean bReturnValueSyntax) { - this.preparedSQLText = preparedSQLText; + ParsedSQLCacheItem(String processedSQL, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + this.processedSQL = processedSQL; this.parameterCount = parameterCount; this.procedureName = procedureName; this.bReturnValueSyntax = bReturnValueSyntax; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 30237a67a..e368149ec 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -57,7 +57,7 @@ import org.ietf.jgss.GSSException; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder; import com.googlecode.concurrentlinkedhashmap.EvictionListener; /** @@ -5619,7 +5619,7 @@ public void onEviction(SQLServerPreparedStatement.Sha1HashKey key, PreparedState /** Add cache entry for prepared statement metadata*/ final void cachePreparedStatementExecuteSqlUse(SQLServerPreparedStatement.Sha1HashKey key) { // Caching turned off? - if(0 >= this.getStatementPoolingCacheSize()) + if(!this.isStatementPoolingEnabled()) return; PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); @@ -5636,7 +5636,7 @@ final void cachePreparedStatementExecuteSqlUse(SQLServerPreparedStatement.Sha1Ha /** Add cache entry for prepared statement metadata*/ final void cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey key, int handle, boolean directSql, SQLServerPreparedStatement statement) { // Caching turned off? - if(0 >= this.getStatementPoolingCacheSize()) + if(!this.isStatementPoolingEnabled()) return; PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); @@ -5655,7 +5655,7 @@ final void cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey k /** Add cache entry for prepared statement metadata*/ final void cacheParameterMetadata(SQLServerPreparedStatement.Sha1HashKey key, SQLServerParameterMetaData metadata, SQLServerPreparedStatement statement) { // Caching turned off? - if(0 >= this.getStatementPoolingCacheSize()) + if(!this.isStatementPoolingEnabled()) return; PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); @@ -5668,18 +5668,16 @@ final void cacheParameterMetadata(SQLServerPreparedStatement.Sha1HashKey key, SQ /** Add cache entry for prepared statement metadata*/ private void cachePreparedStatementMetadata(SQLServerPreparedStatement.Sha1HashKey key, PreparedStatementCacheItem cacheItem) { // Caching turned off? - if(0 >= this.getStatementPoolingCacheSize() || null == cacheItem) + if(!this.isStatementPoolingEnabled() || null == cacheItem || null == key) return; - if(null == this.preparedStatementCache) { + if(null == this.preparedStatementCache) this.preparedStatementCache = new Builder() .maximumWeightedCapacity(this.getStatementPoolingCacheSize()) .listener(new PreparedStatementHandleEvictionListener()) .build(); - } - if(null != key) - this.preparedStatementCache.put(key, cacheItem); + this.preparedStatementCache.put(key, cacheItem); } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index a9847b5cb..ff86663a0 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -54,13 +54,10 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS private static final int BATCH_STATEMENT_DELIMITER_TDS_72 = 0xFF; final int nBatchStatementDelimiter = BATCH_STATEMENT_DELIMITER_TDS_72; - /** the user's prepared sql syntax */ - //private String sqlCommand; - /** The prepared type definitions */ private String preparedTypeDefinitions; - /** Processed SQL statement text that will be executed, may not be same as what user initially passed. */ + /** Processed SQL statement text, may not be same as what user initially passed. */ final String userSQL; /** SQL statement with expanded parameter tokens */ @@ -145,7 +142,7 @@ private void resetPrepStmtHandle() { /** Size of the parsed SQL-text metadata cache */ - static final private int parsedSQLCacheSize = 100; + static final private int PARSED_SQL_CACHE_SIZE = 100; static class Sha1HashKey { private byte[] bytes; @@ -173,16 +170,16 @@ private java.security.MessageDigest getSha1Digest() { } } - /** Cache of prepared statement meta data */ + /** Cache of parsed SQL meta data */ static private ConcurrentLinkedHashMap parsedSQLCache; static { parsedSQLCache = new Builder() - .maximumWeightedCapacity(parsedSQLCacheSize) + .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE) .build(); } - /** Get prepared statement cache entry if exists */ + /** Get parsed SQL cache entry if exists */ static ParsedSQLCacheItem getCachedParsedSQLMetadata(Sha1HashKey key) { if(null == key) return null; @@ -190,7 +187,7 @@ static ParsedSQLCacheItem getCachedParsedSQLMetadata(Sha1HashKey key) { return parsedSQLCache.get(key); } - /** Add cache entry for prepared statement metadata*/ + /** Add cache entry for parsed SQL metadata*/ static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql, Sha1HashKey key) throws SQLServerException { JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); @@ -202,7 +199,7 @@ static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql, Sha1HashKe // Cache this entry. ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); - if(null != initialSql) + if(null != key) parsedSQLCache.put(key, cacheItem); return cacheItem; @@ -246,17 +243,17 @@ String getClassNameInternal() { // Check for cached SQL metadata. ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(cacheKey); - // No cached meta data found, parse. + // No cached meta data found, parse and cache. if(null == cacheItem) cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql, cacheKey); - // Retrieve from cache item. + // Retrieve meta data from cache item. procedureName = cacheItem.procedureName; bReturnValueSyntax = cacheItem.bReturnValueSyntax; - userSQL = cacheItem.preparedSQLText; + userSQL = cacheItem.processedSQL; initParams(cacheItem.parameterCount); - // See if existing handle can be re-used. + // See if existing prepared statement handle can be re-used. handleUsingCachedStmtHandle(); } @@ -290,7 +287,8 @@ private void closePreparedHandle() { !this.connection.isStatementPoolingEnabled() // No caching || ( null != cachedPreparedStatementHandle // Cache ref. exists - && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache + && cachedPreparedStatementHandle.hasHandle()// Has a handle to discard + && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache, will not be re-used by other stmts. && cachedPreparedStatementHandle.discardIfHandleNotReferenced() // Not used by any other statements. ) ) { @@ -621,6 +619,7 @@ else if (EXECUTE_UPDATE == executeMethod && null != resultSet) { } } + /** Lookup existing prepared statement handle in cache and re-use if available. */ private void handleUsingCachedStmtHandle() { if(!hasPreparedStatementHandle()) { // Check for cached handle. @@ -629,7 +628,7 @@ private void handleUsingCachedStmtHandle() { // If handle was found then re-use. if(null != cachedHandle && (cachedHandle.hasHandle() || cachedHandle.hasExecutedSpExecuteSql)) { - // If existing handle was found use it and specify no need for prepare. + // If existing handle was found use it. if(cachedHandle.hasHandle() && cachedHandle.incrementHandleRefCountAndVerifyNotInvalidated(this)) { setPrepStmtHandle(cachedHandle.handle, userSQL, false); } @@ -1050,11 +1049,7 @@ else if (resultSet != null) { */ /* L0 */ private ResultSet buildExecuteMetaData() throws SQLServerException { String fmtSQL = userSQL; - /* - if (fmtSQL.indexOf(LEFT_CURLY_BRACKET) >= 0) { - fmtSQL = userSQL; - } - */ + ResultSet emptyResultSet = null; try { fmtSQL = replaceMarkerWithNull(fmtSQL); diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index ec43e4106..e681dd852 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -773,7 +773,7 @@ private String ensureSQLSyntax(String sql) throws SQLServerException { // Retrieve from cache item. procedureName = cacheItem.procedureName; - return cacheItem.preparedSQLText; + return cacheItem.processedSQL; } return sql; From baf554444dba1f7374ebcb82efe0f77ab4a8dae6 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Sat, 6 May 2017 23:53:30 -0500 Subject: [PATCH 13/34] Fixed eviction bug and added eviction test case. --- .../sqlserver/jdbc/SQLServerConnection.java | 21 ++++-- .../jdbc/SQLServerPreparedStatement.java | 40 ++++++----- .../unit/statement/PreparedStatementTest.java | 71 +++++++++++++++++++ 3 files changed, 110 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index e368149ec..b80acf77f 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -5573,6 +5573,17 @@ public int getStatementPoolingCacheSize() { return this.statementPoolingCacheSize; } + /** + * Returns the current number of pooled prepared statements. + * @return Returns the current setting per the description. + */ + public int getStatementPoolingCacheEntryCount() { + if(null == this.preparedStatementCache) + return 0; + else + return this.preparedStatementCache.size(); + } + /** * Whether statement pooling is enabled or not for this connection. * @return Returns the current setting per the description. @@ -5610,8 +5621,8 @@ public void onEviction(SQLServerPreparedStatement.Sha1HashKey key, PreparedState // Only discard if not referenced. if(cacheItem.hasHandle() && cacheItem.discardIfHandleNotReferenced()) { cacheItem.connection.enqueuePreparedStatementDiscardItem(cacheItem.handle, cacheItem.handleIsDirectSql); - cacheItem.connection.handlePreparedStatementDiscardActions(false); - } + // Do not run discard actions here! Can interfere with executing statement. + } } } } @@ -5634,10 +5645,10 @@ final void cachePreparedStatementExecuteSqlUse(SQLServerPreparedStatement.Sha1Ha } /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey key, int handle, boolean directSql, SQLServerPreparedStatement statement) { + final PreparedStatementCacheItem cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey key, int handle, boolean directSql, SQLServerPreparedStatement statement) { // Caching turned off? if(!this.isStatementPoolingEnabled()) - return; + return null; PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); @@ -5650,6 +5661,8 @@ final void cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey k this.cachePreparedStatementMetadata(key, cacheItem); } + + return cacheItem; } /** Add cache entry for prepared statement metadata*/ diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index ff86663a0..a56f3b348 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -122,8 +122,8 @@ private void setPrepStmtHandle(int handle, String sql, boolean cache) { prepStmtHandle = handle; - if(cache) - connection.cachePreparedStatementHandle(cacheKey, handle, executedSqlDirectly, this); + if(cache) + cachedPreparedStatementHandle = connection.cachePreparedStatementHandle(cacheKey, handle, executedSqlDirectly, this); } /** Resets the server handle for this prepared statement to no handle. @@ -276,27 +276,31 @@ private void closePreparedHandle() { final int handleToClose = getPreparedStatementHandle(); resetPrepStmtHandle(); + // Handle unprepare actions through batching @ connection level. + if (null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) { + cachedPreparedStatementHandle.decrementHandleRefCount(); + } + + boolean notReferencedByStatementCache = !this.connection.isStatementPoolingEnabled() // No caching + || null == cachedPreparedStatementHandle // No cache reference + || ( + null != cachedPreparedStatementHandle // Cache ref. exists + && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache, will not be re-used by other stmts. + && cachedPreparedStatementHandle.discardIfHandleNotReferenced() // Not used by any other statements. + ); + // Using batched clean-up? If not, use old method of calling sp_unprepare. if(1 < connection.getServerPreparedStatementDiscardThreshold()) { - // Handle unprepare actions through batching @ connection level. - // Use this only if statement caching is off, otherwise this will be called by statement cache invalidation. - if(null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) - cachedPreparedStatementHandle.decrementHandleRefCount(); - - if( - !this.connection.isStatementPoolingEnabled() // No caching - || ( - null != cachedPreparedStatementHandle // Cache ref. exists - && cachedPreparedStatementHandle.hasHandle()// Has a handle to discard - && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache, will not be re-used by other stmts. - && cachedPreparedStatementHandle.discardIfHandleNotReferenced() // Not used by any other statements. - ) - ) { + // Handle properly with statement caching . + if(notReferencedByStatementCache) { connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); - connection.handlePreparedStatementDiscardActions(false); } } - else { + + // Always run any outstanding discard actions as statement pooling always uses batched sp_unprepare. + connection.handlePreparedStatementDiscardActions(false); + + if(notReferencedByStatementCache) { // Non batched behavior (same as pre batch impl.) if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) getStatementLogger().finer(this + ": Closing PreparedHandle:" + handleToClose); diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index b0561547d..087b1bcfd 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -190,6 +190,77 @@ public void testStatementPooling() throws SQLException { } } + /** + * Test handling of eviction from statement pooling for prepared statements. + * + * @throws SQLException + */ + @Test + public void testStatementPoolingEviction() throws SQLException { + // Make sure correct settings are used. + SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); + SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); + + for (int testNo = 0; testNo < 2; ++testNo) { + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + + int cacheSize = 10; + int discardedStatementCount = testNo == 0 ? 5 /*batched unprepares*/ : 0 /*regular unprepares*/; + + con.setStatementPoolingCacheSize(cacheSize); + con.setServerPreparedStatementDiscardThreshold(discardedStatementCount); + + String lookupUniqueifier = UUID.randomUUID().toString(); + String query = String.format("/*statementpoolingevictiontest_%s*/SELECT * FROM sys.tables; -- ", lookupUniqueifier); + + // Add new statements to fill up the statement pool. + for(int i = 0; i < cacheSize; ++i) { + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + new Integer(i).toString())) { + pstmt.execute(); // sp_executesql + pstmt.execute(); // sp_prepexec, actual handle created and cached. + } + // Make sure no handles in discard queue (still only in statement pool). + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + } + + // No discarded handles yet, all in statement pool. + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + + // Add new statements to fill up the statement discard action queue + // (new statement pushes existing statement from pool into discard + // action queue). + for(int i = cacheSize; i < cacheSize + 5; ++i) { + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + new Integer(i).toString())) { + pstmt.execute(); // sp_executesql + pstmt.execute(); // sp_prepexec, actual handle created and cached. + } + // If we use discard queue handles should start going into discard queue. + if(0 == testNo) + assertNotSame(0, con.getDiscardedServerPreparedStatementCount()); + else + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + } + + // If we use it, now discard queue should be "full". + if(0 == testNo) + assertSame(discardedStatementCount, con.getDiscardedServerPreparedStatementCount()); + else + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + + // Adding one more statement should cause one more pooled statement to be invalidated and + // discarding actions should be executed (i.e. sp_unprepare batch), clearing out the discard + // action queue. + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_executesql + pstmt.execute(); // sp_prepexec, actual handle created and cached. + } + + // Discard queue should now be empty. + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + } + } + } + /** * Test handling of the two configuration knobs related to prepared statement handling. * From 8f50fb75b7ce73dfb871e8ff6034b35c52cd6cef Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Sun, 7 May 2017 16:38:45 -0500 Subject: [PATCH 14/34] Additional test case + ability to change pool size --- .../sqlserver/jdbc/SQLServerConnection.java | 9 +++++- .../unit/statement/PreparedStatementTest.java | 28 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index b80acf77f..36f7e8c97 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -5597,7 +5597,14 @@ public boolean isStatementPoolingEnabled() { * @value The new cache size. */ public void setStatementPoolingCacheSize(int value) { - this.statementPoolingCacheSize = value; + if (value != this.statementPoolingCacheSize) { + value = Math.max(0, value); + this.statementPoolingCacheSize = value; + + if (null != this.preparedStatementCache) { + this.preparedStatementCache.setCapacity(value); + } + } } /** Get prepared statement cache entry if exists */ diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 087b1bcfd..0283c3618 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -214,7 +214,7 @@ public void testStatementPoolingEviction() throws SQLException { String query = String.format("/*statementpoolingevictiontest_%s*/SELECT * FROM sys.tables; -- ", lookupUniqueifier); // Add new statements to fill up the statement pool. - for(int i = 0; i < cacheSize; ++i) { + for (int i = 0; i < cacheSize; ++i) { try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + new Integer(i).toString())) { pstmt.execute(); // sp_executesql pstmt.execute(); // sp_prepexec, actual handle created and cached. @@ -229,7 +229,7 @@ public void testStatementPoolingEviction() throws SQLException { // Add new statements to fill up the statement discard action queue // (new statement pushes existing statement from pool into discard // action queue). - for(int i = cacheSize; i < cacheSize + 5; ++i) { + for (int i = cacheSize; i < cacheSize + 5; ++i) { try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + new Integer(i).toString())) { pstmt.execute(); // sp_executesql pstmt.execute(); // sp_prepexec, actual handle created and cached. @@ -242,7 +242,7 @@ public void testStatementPoolingEviction() throws SQLException { } // If we use it, now discard queue should be "full". - if(0 == testNo) + if (0 == testNo) assertSame(discardedStatementCount, con.getDiscardedServerPreparedStatementCount()); else assertSame(0, con.getDiscardedServerPreparedStatementCount()); @@ -257,6 +257,28 @@ public void testStatementPoolingEviction() throws SQLException { // Discard queue should now be empty. assertSame(0, con.getDiscardedServerPreparedStatementCount()); + + // Set statement pool size to 0 and verify statements get discarded. + int statementsInCache = con.getStatementPoolingCacheEntryCount(); + con.setStatementPoolingCacheSize(0); + assertSame(0, con.getStatementPoolingCacheEntryCount()); + + if(0 == testNo) + // Verify statements moved over to discard action queue. + assertSame(statementsInCache, con.getDiscardedServerPreparedStatementCount()); + + // Run discard actions (otherwise run on pstmt.close) + con.closeDiscardedServerPreparedStatements(); + + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + + // Verify new statement does not go into cache (since cache is now off) + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + pstmt.execute(); // sp_executesql + pstmt.execute(); // sp_prepexec, actual handle created and cached. + + assertSame(0, con.getStatementPoolingCacheEntryCount()); + } } } } From 12f94cfec42ee35060573d30aedeb5201cccbb00 Mon Sep 17 00:00:00 2001 From: tobiast Date: Mon, 8 May 2017 15:51:35 -0700 Subject: [PATCH 15/34] Merge of @brettwooldridge's changes --- pom.xml | 2 +- .../sqlserver/jdbc/SQLServerConnection.java | 401 ++++++++++-------- .../jdbc/SQLServerPreparedStatement.java | 229 +++------- .../sqlserver/jdbc/SQLServerStatement.java | 14 +- .../unit/statement/PreparedStatementTest.java | 4 +- 5 files changed, 289 insertions(+), 361 deletions(-) diff --git a/pom.xml b/pom.xml index c5d123987..c3a831105 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,7 @@ com.zaxxer HikariCP - 2.6.0 + 2.6.1 test diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 36f7e8c97..ecee2fdc6 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -103,8 +103,8 @@ public class SQLServerConnection implements ISQLServerConnection { private Boolean enablePrepareOnFirstPreparedStatementCall = null; // Current limit for this particular connection. // Handle the actual queue of discarded prepared statements. - private ConcurrentLinkedQueue discardedPreparedStatementHandles = new ConcurrentLinkedQueue(); - private AtomicInteger discardedPreparedStatementHandleQueueCount = new AtomicInteger(0); + private ConcurrentLinkedQueue discardedPreparedStatementHandles = new ConcurrentLinkedQueue(); + private AtomicInteger discardedPreparedStatementHandleCount = new AtomicInteger(0); private boolean fedAuthRequiredByUser = false; private boolean fedAuthRequiredPreLoginResponse = false; @@ -119,61 +119,131 @@ public class SQLServerConnection implements ISQLServerConnection { private SqlFedAuthToken fedAuthToken = null; + static class Sha1HashKey { + private byte[] bytes; + + Sha1HashKey(String s) { + bytes = getSha1Digest().digest(s.getBytes()); + } + + public boolean equals(Object obj) { + if (!(obj instanceof Sha1HashKey)) + return false; + + return java.util.Arrays.equals(bytes, ((Sha1HashKey)obj).bytes); + } + + public int hashCode() { + return java.util.Arrays.hashCode(bytes); + } + + private java.security.MessageDigest getSha1Digest() { + try { + return java.security.MessageDigest.getInstance("SHA-1"); + } + catch (final java.security.NoSuchAlgorithmException e) { + // This is not theoretically possible, but we're forced to catch it anyway + throw new RuntimeException(e); + } + } + } + + /** Size of the parsed SQL-text metadata cache */ + static final private int PARSED_SQL_CACHE_SIZE = 100; + + /** Cache of parsed SQL meta data */ + static private ConcurrentLinkedHashMap parsedSQLCache; + + static { + parsedSQLCache = new Builder() + .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE) + .build(); + } + + /** Get prepared statement cache entry if exists, if not parse and create a new one */ + static ParsedSQLCacheItem getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, String sql) throws SQLServerException { + ParsedSQLCacheItem cacheItem = parsedSQLCache.get(key); + if (null == cacheItem) { + JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); + + String parsedSql = translator.translate(sql); + String procName = translator.getProcedureName(); // may return null + boolean returnValueSyntax = translator.hasReturnValueSyntax(); + int paramCount = countParams(parsedSql); + + cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); + parsedSQLCache.putIfAbsent(key, cacheItem); + } + + return cacheItem; + } + + /** + * Find statement parameters. + * + * @param sql + * SQL text to parse for number of parameters to intialize. + */ + private static int countParams(String sql) { + int nParams = 0; + + // Figure out the expected number of parameters by counting the + // parameter placeholders in the SQL string. + int offset = -1; + while ((offset = ParameterUtils.scanSQLForChar('?', sql, ++offset)) < sql.length()) + ++nParams; + + return nParams; + } + /** * Used to keep track of an individual handle ready for un-prepare. */ final class PreparedStatementCacheItem { - int handle; - boolean hasExecutedSpExecuteSql; - boolean handleIsDirectSql; - boolean evictedFromCache; - SQLServerConnection connection; - private AtomicInteger refCount = new AtomicInteger(1); - SQLServerParameterMetaData parameterMetadata; - - PreparedStatementCacheItem(int handle, boolean handleIsDirectSql, boolean hasExecutedSpExecuteSql, SQLServerParameterMetaData parameterMetadata, SQLServerConnection connection) { - this.handle = handle; - this.handleIsDirectSql = handleIsDirectSql; + private final PreparedStatementHandle statementHandle = new PreparedStatementHandle(); + private volatile SQLServerParameterMetaData parameterMetadata; + private volatile boolean hasExecutedSpExecuteSql; + volatile boolean evictedFromCache; + + boolean hasExecutedSpExecuteSql() { + return hasExecutedSpExecuteSql; + } + + void setHasExecutedSpExecuteSql(boolean hasExecutedSpExecuteSql) { this.hasExecutedSpExecuteSql = hasExecutedSpExecuteSql; - this.connection = connection; - this.parameterMetadata = parameterMetadata; } boolean hasHandle() { - return 0 < this.handle; + return 0 < statementHandle.getHandle(); } + int getHandleAndIncrementRefCount() { + statementHandle.incrementRefCount(); + return statementHandle.getHandle(); + } + + void setHandle(int handle, boolean isDirectSql) { + statementHandle.setHandle(handle, isDirectSql); + } + boolean hasParameterMetadata() { return null != this.parameterMetadata; } - // Returns false if handle is not re-usable. - boolean incrementHandleRefCountAndVerifyNotInvalidated(SQLServerPreparedStatement statement) { - // If refcount is negative the handle has been killed. - if(0 > this.refCount.getAndIncrement()) { - this.refCount.getAndDecrement(); // Reduce again. - return false; - } - else { - statement.cachedPreparedStatementHandle = this; - return true; - } + SQLServerParameterMetaData getParameterMetadata() { + return parameterMetadata; } - - boolean discardIfHandleNotReferenced() { - // If refcount is zero or negative the handle can be killed. - if(1 > this.refCount.getAndDecrement()) { - return true; - } - else { - // In use. - this.refCount.getAndIncrement(); // Return back. - return false; - } + + void setParameterMetadata(SQLServerParameterMetaData metadata) { + parameterMetadata = metadata; + } + + int decrementHandleRefCount() { + return statementHandle.decrementRefCount(); } - void decrementHandleRefCount() { - this.refCount.decrementAndGet(); + int getHandleRefCount() { + return statementHandle.getRefCount(); } } @@ -181,7 +251,7 @@ void decrementHandleRefCount() { private int statementPoolingCacheSize = 10; /** Cache of prepared statement handles */ - private ConcurrentLinkedHashMap preparedStatementCache; + private ConcurrentLinkedHashMap preparedStatementCache; SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; @@ -788,6 +858,14 @@ final boolean attachConnId() { connectionlogger.severe(message); throw new UnsupportedOperationException(message); } + + // Caching turned off? + if (isStatementPoolingEnabled()) { + preparedStatementCache = new Builder() + .maximumWeightedCapacity(getStatementPoolingCacheSize()) + .listener(new PreparedStatementCacheEvictionListener()) + .build(); + } } void setFailoverPartnerServerProvided(String partner) { @@ -5317,16 +5395,47 @@ static synchronized long getColumnEncryptionKeyCacheTtl() { /** - * Used to keep track of an individual handle ready for un-prepare. + * Used to keep track of an individual prepared statement handle on the server side. */ - private final class PreparedStatementDiscardItem { + static class PreparedStatementHandle { + private final AtomicInteger handle; + private final AtomicInteger handleRefCount = new AtomicInteger(); + private boolean isDirectSql; + + PreparedStatementHandle() { + handle = new AtomicInteger(); + } + + PreparedStatementHandle(int handle, boolean isDirectSql) { + this.handle = new AtomicInteger(handle); + this.isDirectSql = isDirectSql; + } + + int getHandle() { + return handle.get(); + } + + void setHandle(int handle, boolean isDirectSql) { + if (handleRefCount.compareAndSet(0, 1)) { + this.handle.compareAndSet(0, handle); + this.isDirectSql = isDirectSql; + } + } + + public boolean isDirectSql() { + return isDirectSql; + } + + public int getRefCount() { + return handleRefCount.get(); + } - int handle; - boolean directSql; + public void incrementRefCount() { + this.handleRefCount.incrementAndGet(); + } - PreparedStatementDiscardItem(int handle, boolean directSql) { - this.handle = handle; - this.directSql = directSql; + public int decrementRefCount() { + return this.handleRefCount.decrementAndGet(); } } @@ -5334,18 +5443,16 @@ private final class PreparedStatementDiscardItem { /** * Enqueue a discarded prepared statement handle to be clean-up on the server. * - * @param handle - * The prepared statement handle - * @param directSql - * Whether the statement handle is direct SQL (true) or a cursor (false) + * @param statementHandle + * The prepared statement handle that should be scheduled for unprepare. */ - final void enqueuePreparedStatementDiscardItem(int handle, boolean directSql) { - if (this.getConnectionLogger().isLoggable(java.util.logging.Level.FINER)) - this.getConnectionLogger().finer(this + ": Adding PreparedHandle to queue for un-prepare:" + handle); + final void enqueueUnprepareStatementHandle(PreparedStatementHandle statementHandle) { + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.finer(this + ": Adding PreparedHandle to queue for un-prepare:" + statementHandle.getHandle()); // Add the new handle to the discarding queue and find out current # enqueued. - this.discardedPreparedStatementHandles.add(new PreparedStatementDiscardItem(handle, directSql)); - this.discardedPreparedStatementHandleQueueCount.incrementAndGet(); + this.discardedPreparedStatementHandles.add(statementHandle); + this.discardedPreparedStatementHandleCount.incrementAndGet(); } @@ -5355,7 +5462,7 @@ final void enqueuePreparedStatementDiscardItem(int handle, boolean directSql) { * @return Returns the current value per the description. */ public int getDiscardedServerPreparedStatementCount() { - return this.discardedPreparedStatementHandleQueueCount.get(); + return this.discardedPreparedStatementHandleCount.get(); } /** @@ -5370,7 +5477,7 @@ public void closeDiscardedServerPreparedStatements() { */ private final void cleanupPreparedStatementDiscardActions() { this.discardedPreparedStatementHandles.clear(); - this.discardedPreparedStatementHandleQueueCount.set(0); + this.discardedPreparedStatementHandleCount.set(0); } /** @@ -5457,7 +5564,7 @@ static public int getInitialDefaultServerPreparedStatementDiscardThreshold() { * @return Returns the current setting per the description. */ static public int getDefaultServerPreparedStatementDiscardThreshold() { - if(0 > defaultServerPreparedStatementDiscardThreshold) + if (0 > defaultServerPreparedStatementDiscardThreshold) return getInitialDefaultServerPreparedStatementDiscardThreshold(); else return defaultServerPreparedStatementDiscardThreshold; @@ -5474,7 +5581,7 @@ static public int getDefaultServerPreparedStatementDiscardThreshold() { * Changes the setting per the description. */ static public void setDefaultServerPreparedStatementDiscardThreshold(int value) { - defaultServerPreparedStatementDiscardThreshold = value; + defaultServerPreparedStatementDiscardThreshold = Math.max(0, value); } /** @@ -5487,7 +5594,7 @@ static public void setDefaultServerPreparedStatementDiscardThreshold(int value) * @return Returns the current setting per the description. */ public int getServerPreparedStatementDiscardThreshold() { - if(0 > this.serverPreparedStatementDiscardThreshold) + if (0 > this.serverPreparedStatementDiscardThreshold) return getDefaultServerPreparedStatementDiscardThreshold(); else return this.serverPreparedStatementDiscardThreshold; @@ -5503,7 +5610,7 @@ public int getServerPreparedStatementDiscardThreshold() { * Changes the setting per the description. */ public void setServerPreparedStatementDiscardThreshold(int value) { - this.serverPreparedStatementDiscardThreshold = value; + this.serverPreparedStatementDiscardThreshold = Math.max(0, value); } /** @@ -5514,53 +5621,51 @@ public void setServerPreparedStatementDiscardThreshold(int value) { */ final void handlePreparedStatementDiscardActions(boolean force) { // Skip out if session is unavailable to adhere to previous non-batched behavior. - if (this.isSessionUnAvailable()) + if (isSessionUnAvailable()) return; - final int threshold = this.getServerPreparedStatementDiscardThreshold(); - - // Find out current # enqueued, if force, make sure it always exceeds threshold. - int count = force ? threshold + 1 : this.getDiscardedServerPreparedStatementCount(); + final int threshold = getServerPreparedStatementDiscardThreshold(); // Met threshold to clean-up? - if(threshold < count) { + if (force || threshold < getDiscardedServerPreparedStatementCount()) { - PreparedStatementDiscardItem prepStmtDiscardAction = this.discardedPreparedStatementHandles.poll(); - if(null != prepStmtDiscardAction) { - int handlesRemoved = 0; + // Create batch of sp_unprepare statements. + StringBuilder sql = new StringBuilder(threshold * 32/*EXEC sp_cursorunprepare++;*/); - // Create batch of sp_unprepare statements. - StringBuilder sql = new StringBuilder(count * 32/*EXEC sp_cursorunprepare++;*/); + // Build the string containing no more than the # of handles to remove. + // Note that sp_unprepare can fail if the statement is already removed. + // However, the server will only abort that statement and continue with + // the remaining clean-up. + int handlesRemoved = 0; + PreparedStatementHandle statementHandle = null; - // Build the string containing no more than the # of handles to remove. - // Note that sp_unprepare can fail if the statement is already removed. - // However, the server will only abort that statement and continue with - // the remaining clean-up. - do { - ++handlesRemoved; + while (null != (statementHandle = discardedPreparedStatementHandles.poll())){ + if (0 < statementHandle.getRefCount()) + continue; // Will get discarded when remaining prepared statements get closed. - sql.append(prepStmtDiscardAction.directSql ? "EXEC sp_unprepare " : "EXEC sp_cursorunprepare ") - .append(prepStmtDiscardAction.handle) - .append(';'); - } while (null != (prepStmtDiscardAction = this.discardedPreparedStatementHandles.poll())); - - try { - // Execute the batched set. - try(Statement stmt = this.createStatement()) { - stmt.execute(sql.toString()); - } + ++handlesRemoved; + + sql.append(statementHandle.isDirectSql() ? "EXEC sp_unprepare " : "EXEC sp_cursorunprepare ") + .append(statementHandle.getHandle()) + .append(';'); + } - if (this.getConnectionLogger().isLoggable(java.util.logging.Level.FINER)) - this.getConnectionLogger().finer(this + ": Finished un-preparing handle count:" + handlesRemoved); - } - catch(SQLException e) { - if (this.getConnectionLogger().isLoggable(java.util.logging.Level.FINER)) - this.getConnectionLogger().log(Level.FINER, this + ": Error batch-closing at least one prepared handle", e); + try { + // Execute the batched set. + try(Statement stmt = this.createStatement()) { + stmt.execute(sql.toString()); } - - // Decrement threshold counter - this.discardedPreparedStatementHandleQueueCount.addAndGet(-handlesRemoved); + + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.finer(this + ": Finished un-preparing handle count:" + handlesRemoved); } + catch(SQLException e) { + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.log(Level.FINER, this + ": Error batch-closing at least one prepared handle", e); + } + + // Decrement threshold counter + this.discardedPreparedStatementHandleCount.addAndGet(-handlesRemoved); } } @@ -5607,98 +5712,40 @@ public void setStatementPoolingCacheSize(int value) { } } - /** Get prepared statement cache entry if exists */ - final PreparedStatementCacheItem getCachedPreparedStatementMetadata(SQLServerPreparedStatement.Sha1HashKey key) { - if(null == this.preparedStatementCache) + /** Get or create prepared statement cache entry if statement pooling is enabled */ + final PreparedStatementCacheItem borrowCachedPreparedStatementMetadata(Sha1HashKey key) { + if(!isStatementPoolingEnabled()) return null; - if(null == key) - return null; - else - return this.preparedStatementCache.get(key); + PreparedStatementCacheItem cacheItem = preparedStatementCache.get(key); + if (null == cacheItem) { + cacheItem = new PreparedStatementCacheItem(); + preparedStatementCache.putIfAbsent(key, cacheItem); + } + + return cacheItem; + } + + /** Return prepared statement cache entry so it can be un-prepared. */ + final void returnCachedPreparedStatementMetadata(PreparedStatementCacheItem cacheItem) { + if (0 >= cacheItem.decrementHandleRefCount() && cacheItem.evictedFromCache && cacheItem.hasHandle()) + enqueueUnprepareStatementHandle(cacheItem.statementHandle); } // Handle closing handles when removed from cache. - final class PreparedStatementHandleEvictionListener - implements EvictionListener { - public void onEviction(SQLServerPreparedStatement.Sha1HashKey key, PreparedStatementCacheItem cacheItem) { + final class PreparedStatementCacheEvictionListener implements EvictionListener { + public void onEviction(Sha1HashKey key, PreparedStatementCacheItem cacheItem) { if(null != cacheItem) { cacheItem.evictedFromCache = true; // Mark as evicted from cache. // Only discard if not referenced. - if(cacheItem.hasHandle() && cacheItem.discardIfHandleNotReferenced()) { - cacheItem.connection.enqueuePreparedStatementDiscardItem(cacheItem.handle, cacheItem.handleIsDirectSql); + if(cacheItem.hasHandle() && 0 >= cacheItem.getHandleRefCount()) { + enqueueUnprepareStatementHandle(cacheItem.statementHandle); // Do not run discard actions here! Can interfere with executing statement. } } } } - - /** Add cache entry for prepared statement metadata*/ - final void cachePreparedStatementExecuteSqlUse(SQLServerPreparedStatement.Sha1HashKey key) { - // Caching turned off? - if(!this.isStatementPoolingEnabled()) - return; - - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); - - if(null != cacheItem) - cacheItem.hasExecutedSpExecuteSql = true; - else { - cacheItem = new PreparedStatementCacheItem(0, false, true, null, this); - - this.cachePreparedStatementMetadata(key, cacheItem); - } - } - - /** Add cache entry for prepared statement metadata*/ - final PreparedStatementCacheItem cachePreparedStatementHandle(SQLServerPreparedStatement.Sha1HashKey key, int handle, boolean directSql, SQLServerPreparedStatement statement) { - // Caching turned off? - if(!this.isStatementPoolingEnabled()) - return null; - - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); - - if(null != cacheItem) { - cacheItem.handle = handle; - cacheItem.handleIsDirectSql = directSql; - } - else { - cacheItem = new PreparedStatementCacheItem(handle, directSql, false, null, this); - - this.cachePreparedStatementMetadata(key, cacheItem); - } - - return cacheItem; - } - - /** Add cache entry for prepared statement metadata*/ - final void cacheParameterMetadata(SQLServerPreparedStatement.Sha1HashKey key, SQLServerParameterMetaData metadata, SQLServerPreparedStatement statement) { - // Caching turned off? - if(!this.isStatementPoolingEnabled()) - return; - - PreparedStatementCacheItem cacheItem = this.getCachedPreparedStatementMetadata(key); - if(null != cacheItem) - cacheItem.parameterMetadata = metadata; - else - this.cachePreparedStatementMetadata(key, new PreparedStatementCacheItem(0, false, false, metadata, this)); - } - - /** Add cache entry for prepared statement metadata*/ - private void cachePreparedStatementMetadata(SQLServerPreparedStatement.Sha1HashKey key, PreparedStatementCacheItem cacheItem) { - // Caching turned off? - if(!this.isStatementPoolingEnabled() || null == cacheItem || null == key) - return; - - if(null == this.preparedStatementCache) - this.preparedStatementCache = new Builder() - .maximumWeightedCapacity(this.getStatementPoolingCacheSize()) - .listener(new PreparedStatementHandleEvictionListener()) - .build(); - - this.preparedStatementCache.put(key, cacheItem); - } } // Helper class for security manager functions used by SQLServerConnection class. diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index a56f3b348..473f6b8f4 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -8,6 +8,8 @@ package com.microsoft.sqlserver.jdbc; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getOrCreateCachedParsedSQLMetadata; + import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; @@ -28,8 +30,9 @@ import java.util.Vector; import java.util.logging.Level; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;; +import com.microsoft.sqlserver.jdbc.SQLServerConnection.PreparedStatementCacheItem; +import com.microsoft.sqlserver.jdbc.SQLServerConnection.PreparedStatementHandle; +import com.microsoft.sqlserver.jdbc.SQLServerConnection.Sha1HashKey; /** * SQLServerPreparedStatement provides JDBC prepared statement functionality. SQLServerPreparedStatement provides methods for the user to supply @@ -67,7 +70,7 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS private boolean isExecutedAtLeastOnce = false; /** Reference to cache item for statement pooling. Only used to decrement ref count on statement close. */ - SQLServerConnection.PreparedStatementCacheItem cachedPreparedStatementHandle; + private final PreparedStatementCacheItem cachedPreparedStatementItem; /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set @@ -111,21 +114,6 @@ private boolean hasPreparedStatementHandle() { return 0 < prepStmtHandle; } - /** Sets the server handle for this prepared statement. - * - * @handle - * @sql - * @cache - */ - private void setPrepStmtHandle(int handle, String sql, boolean cache) { - assert 0 < handle; - - prepStmtHandle = handle; - - if(cache) - cachedPreparedStatementHandle = connection.cachePreparedStatementHandle(cacheKey, handle, executedSqlDirectly, this); - } - /** Resets the server handle for this prepared statement to no handle. */ private void resetPrepStmtHandle() { @@ -140,79 +128,11 @@ private void resetPrepStmtHandle() { */ private boolean encryptionMetadataIsRetrieved = false; - - /** Size of the parsed SQL-text metadata cache */ - static final private int PARSED_SQL_CACHE_SIZE = 100; - - static class Sha1HashKey { - private byte[] bytes; - - Sha1HashKey(String s) { - bytes = getSha1Digest().digest(s.getBytes()); - } - - public boolean equals(Object obj) { - return java.util.Arrays.equals(bytes, ((Sha1HashKey)obj).bytes); - } - - public int hashCode() { - return java.util.Arrays.hashCode(bytes); - } - - private java.security.MessageDigest getSha1Digest() { - try { - return java.security.MessageDigest.getInstance("SHA-1"); - } - catch (final java.security.NoSuchAlgorithmException e) { - // This is not theoretically possible, but we're forced to catch it anyway - throw new RuntimeException(e); - } - } - } - - /** Cache of parsed SQL meta data */ - static private ConcurrentLinkedHashMap parsedSQLCache; - - static { - parsedSQLCache = new Builder() - .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE) - .build(); - } - - /** Get parsed SQL cache entry if exists */ - static ParsedSQLCacheItem getCachedParsedSQLMetadata(Sha1HashKey key) { - if(null == key) - return null; - else - return parsedSQLCache.get(key); - } - - /** Add cache entry for parsed SQL metadata*/ - static ParsedSQLCacheItem parseAndCacheSQLMetadata(String initialSql, Sha1HashKey key) throws SQLServerException { - - JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); - - String parsedSql = translator.translate(initialSql); - String procName = translator.getProcedureName(); // may return null - boolean returnValueSyntax = translator.hasReturnValueSyntax(); - int paramCount = countParams(parsedSql); - - // Cache this entry. - ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); - if(null != key) - parsedSQLCache.put(key, cacheItem); - - return cacheItem; - } - // Internal function used in tracing String getClassNameInternal() { return "SQLServerPreparedStatement"; } - /** Key used to lookup this statement in caches. */ - private Sha1HashKey cacheKey; - /** * Create a new prepaed statement. * @@ -235,18 +155,21 @@ String getClassNameInternal() { int nRSConcur, SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException { super(conn, nRSType, nRSConcur, stmtColEncSetting); + + if (null == sql) { + MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_NullValue")); + Object[] msgArgs1 = {"Statement SQL"}; + throw new SQLServerException(form.format(msgArgs1), null); + } + stmtPoolable = true; // Create a cache key for this statement. - cacheKey = new Sha1HashKey(sql); + Sha1HashKey cacheKey = new Sha1HashKey(sql); - // Check for cached SQL metadata. - ParsedSQLCacheItem cacheItem = getCachedParsedSQLMetadata(cacheKey); + // Parse or fetch SQL metadata from cache. + ParsedSQLCacheItem cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); - // No cached meta data found, parse and cache. - if(null == cacheItem) - cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql, cacheKey); - // Retrieve meta data from cache item. procedureName = cacheItem.procedureName; bReturnValueSyntax = cacheItem.bReturnValueSyntax; @@ -254,7 +177,18 @@ String getClassNameInternal() { initParams(cacheItem.parameterCount); // See if existing prepared statement handle can be re-used. - handleUsingCachedStmtHandle(); + cachedPreparedStatementItem = connection.borrowCachedPreparedStatementMetadata(cacheKey); + // If handle was found then re-use. + if(null != cachedPreparedStatementItem && cachedPreparedStatementItem.hasExecutedSpExecuteSql()) { + // If existing handle was found use it. + if(cachedPreparedStatementItem.hasHandle()) { + prepStmtHandle = cachedPreparedStatementItem.getHandleAndIncrementRefCount(); + } + else { + // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. + isExecutedAtLeastOnce = true; + } + } } /** @@ -268,8 +202,8 @@ private void closePreparedHandle() { // the prepared handle. We won't be able to, and it's already closed // on the server anyway. if (connection.isSessionUnAvailable()) { - if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().finer(this + ": Not closing PreparedHandle:" + getPreparedStatementHandle() + "; connection is already closed."); + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.finer(this + ": Not closing PreparedHandle:" + getPreparedStatementHandle() + "; connection is already closed."); } else { isExecutedAtLeastOnce = false; @@ -277,33 +211,17 @@ private void closePreparedHandle() { resetPrepStmtHandle(); // Handle unprepare actions through batching @ connection level. - if (null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasHandle()) { - cachedPreparedStatementHandle.decrementHandleRefCount(); + if (null != cachedPreparedStatementItem) { + connection.returnCachedPreparedStatementMetadata(cachedPreparedStatementItem); } - - boolean notReferencedByStatementCache = !this.connection.isStatementPoolingEnabled() // No caching - || null == cachedPreparedStatementHandle // No cache reference - || ( - null != cachedPreparedStatementHandle // Cache ref. exists - && cachedPreparedStatementHandle.evictedFromCache // Evicted from cache, will not be re-used by other stmts. - && cachedPreparedStatementHandle.discardIfHandleNotReferenced() // Not used by any other statements. - ); - // Using batched clean-up? If not, use old method of calling sp_unprepare. - if(1 < connection.getServerPreparedStatementDiscardThreshold()) { - // Handle properly with statement caching . - if(notReferencedByStatementCache) { - connection.enqueuePreparedStatementDiscardItem(handleToClose, executedSqlDirectly); - } + else if(1 < connection.getServerPreparedStatementDiscardThreshold()) { + connection.enqueueUnprepareStatementHandle(new PreparedStatementHandle(getPreparedStatementHandle(), executedSqlDirectly)); } - - // Always run any outstanding discard actions as statement pooling always uses batched sp_unprepare. - connection.handlePreparedStatementDiscardActions(false); - - if(notReferencedByStatementCache) { + else { // Non batched behavior (same as pre batch impl.) - if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().finer(this + ": Closing PreparedHandle:" + handleToClose); + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.finer(this + ": Closing PreparedHandle:" + handleToClose); final class PreparedHandleClose extends UninterruptableTDSCommand { PreparedHandleClose() { @@ -327,13 +245,16 @@ final boolean doExecute() throws SQLServerException { executeCommand(new PreparedHandleClose()); } catch (SQLServerException e) { - if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().log(Level.FINER, this + ": Error (ignored) closing PreparedHandle:" + handleToClose, e); + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.log(Level.FINER, this + ": Error (ignored) closing PreparedHandle:" + handleToClose, e); } - if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) - getStatementLogger().finer(this + ": Closed PreparedHandle:" + handleToClose); + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) + loggerExternal.finer(this + ": Closed PreparedHandle:" + handleToClose); } + + // Always run any outstanding discard actions as statement pooling always uses batched sp_unprepare. + connection.handlePreparedStatementDiscardActions(false); } } @@ -354,24 +275,6 @@ final void closeInternal() { batchParamValues = null; } - /** - * Find statement parameters. - * - * @param sql - * SQL text to parse for number of parameters to intialize. - */ - static int countParams(String sql) { - int nParams = 0; - - // Figure out the expected number of parameters by counting the - // parameter placeholders in the SQL string. - int offset = -1; - while ((offset = ParameterUtils.scanSQLForChar('?', sql, ++offset)) < sql.length()) - ++nParams; - - return nParams; - } - /** * Intialize the statement parameters. * @@ -623,27 +526,6 @@ else if (EXECUTE_UPDATE == executeMethod && null != resultSet) { } } - /** Lookup existing prepared statement handle in cache and re-use if available. */ - private void handleUsingCachedStmtHandle() { - if(!hasPreparedStatementHandle()) { - // Check for cached handle. - SQLServerConnection.PreparedStatementCacheItem cachedHandle = this.connection.getCachedPreparedStatementMetadata(cacheKey); - - // If handle was found then re-use. - if(null != cachedHandle && (cachedHandle.hasHandle() || cachedHandle.hasExecutedSpExecuteSql)) { - - // If existing handle was found use it. - if(cachedHandle.hasHandle() && cachedHandle.incrementHandleRefCountAndVerifyNotInvalidated(this)) { - setPrepStmtHandle(cachedHandle.handle, userSQL, false); - } - else { - // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. - isExecutedAtLeastOnce = true; - } - } - } - } - /** * Consume the OUT parameter for the statement object itself. * @@ -664,8 +546,11 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { expectPrepStmtHandle = false; Parameter param = new Parameter(Util.shouldHonorAEForParameters(stmtColumnEncriptionSetting, connection)); param.skipRetValStatus(tdsReader); - int prepStmtHandle = param.getInt(tdsReader); - setPrepStmtHandle(prepStmtHandle, userSQL, true); + prepStmtHandle = param.getInt(tdsReader); + + if (null != cachedPreparedStatementItem) + cachedPreparedStatementItem.setHandle(prepStmtHandle, executedSqlDirectly); + param.skipValue(tdsReader, true); if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) getStatementLogger().finer(toString() + ": Setting PreparedHandle:" + prepStmtHandle); @@ -1005,7 +890,8 @@ private boolean doPrepExec(TDSWriter tdsWriter, isExecutedAtLeastOnce = true; // Enable re-use if caching is on by moving to sp_prepexec on next call even from separate instance. - connection.cachePreparedStatementExecuteSqlUse(cacheKey); + if (null != cachedPreparedStatementItem) + cachedPreparedStatementItem.setHasExecutedSpExecuteSql(true); } // Second execution, use prepared statements since we seem to be re-using it. else if(needsPrepare) @@ -2974,21 +2860,16 @@ public final void setNull(int paramIndex, * Per the description. */ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws SQLServerException { - - SQLServerConnection.PreparedStatementCacheItem cacheItem = null; - if( - !forceRefresh - && null != (cacheItem = connection.getCachedPreparedStatementMetadata(cacheKey)) - && cacheItem.hasParameterMetadata() - ) { - return cacheItem.parameterMetadata; + if (!forceRefresh && null != cachedPreparedStatementItem && cachedPreparedStatementItem.hasParameterMetadata()) { + return cachedPreparedStatementItem.getParameterMetadata(); } else { loggerExternal.entering(getClassNameLogging(), "getParameterMetaData"); checkClosed(); SQLServerParameterMetaData pmd = new SQLServerParameterMetaData(this, userSQL); - connection.cacheParameterMetadata(cacheKey, pmd, this); + if(null != cachedPreparedStatementItem) + cachedPreparedStatementItem.setParameterMetadata(pmd); loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index e681dd852..7039d6325 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -8,6 +8,8 @@ package com.microsoft.sqlserver.jdbc; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getOrCreateCachedParsedSQLMetadata; + import java.sql.BatchUpdateException; import java.sql.ResultSet; import java.sql.SQLException; @@ -24,6 +26,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.microsoft.sqlserver.jdbc.SQLServerConnection.Sha1HashKey; + /** * SQLServerStatment provides the basic implementation of JDBC statement functionality. It also provides a number of base class implementation methods * for the JDBC prepared statement and callable Statements. SQLServerStatement's basic role is to execute SQL statements and return update counts and @@ -762,15 +766,11 @@ final void processResponse(TDSReader tdsReader) throws SQLServerException { private String ensureSQLSyntax(String sql) throws SQLServerException { if (sql.indexOf(LEFT_CURLY_BRACKET) >= 0) { - SQLServerPreparedStatement.Sha1HashKey cacheKey = new SQLServerPreparedStatement.Sha1HashKey(sql); + Sha1HashKey cacheKey = new Sha1HashKey(sql); // Check for cached SQL metadata. - ParsedSQLCacheItem cacheItem = SQLServerPreparedStatement.getCachedParsedSQLMetadata(cacheKey); - - // No cached SQL-text meta datafound, parse. - if(null == cacheItem) - cacheItem = SQLServerPreparedStatement.parseAndCacheSQLMetadata(sql, cacheKey); - + ParsedSQLCacheItem cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); + // Retrieve from cache item. procedureName = cacheItem.procedureName; return cacheItem.processedSQL; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 0283c3618..46700a671 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -371,9 +371,9 @@ public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws assertNotSame(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold(), SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); assertNotSame(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall(), SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); - // Verify invalid (negative) change does not stick for threshold. + // Verify invalid (negative) changes are handled correctly. SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(-1); - assertTrue(0 < SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); + assertSame(0, SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); // Verify instance settings. SQLServerConnection conn1 = (SQLServerConnection)DriverManager.getConnection(connectionString); From 3d5ba9b640706e456afd19a35785bb77886c6111 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Mon, 8 May 2017 19:20:28 -0700 Subject: [PATCH 16/34] Update SQLServerPreparedStatement.java --- .../microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 473f6b8f4..638f0cdaf 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -6,7 +6,7 @@ * This program is made available under the terms of the MIT License. See the LICENSE file in the project root for more information. */ -package com.microsoft.sqlserver.jdbc; +package com.microsoft.sqlserver.jdbc; import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getOrCreateCachedParsedSQLMetadata; From 069d2cf702f5dab044b184aa7551d932b3b4d2be Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Mon, 15 May 2017 07:10:51 -0700 Subject: [PATCH 17/34] Cleaned up and simplified ref counting. --- .../sqlserver/jdbc/SQLServerConnection.java | 75 ++++++++++--------- .../jdbc/SQLServerPreparedStatement.java | 22 ++++-- .../unit/statement/PreparedStatementTest.java | 1 - 3 files changed, 56 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index ecee2fdc6..ef7548911 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -217,13 +217,8 @@ boolean hasHandle() { return 0 < statementHandle.getHandle(); } - int getHandleAndIncrementRefCount() { - statementHandle.incrementRefCount(); - return statementHandle.getHandle(); - } - - void setHandle(int handle, boolean isDirectSql) { - statementHandle.setHandle(handle, isDirectSql); + PreparedStatementHandle getPreparedStatementHandle() { + return statementHandle; } boolean hasParameterMetadata() { @@ -237,14 +232,6 @@ SQLServerParameterMetaData getParameterMetadata() { void setParameterMetadata(SQLServerParameterMetaData metadata) { parameterMetadata = metadata; } - - int decrementHandleRefCount() { - return statementHandle.decrementRefCount(); - } - - int getHandleRefCount() { - return statementHandle.getRefCount(); - } } /** Size of the prepared statement handle cache */ @@ -5398,44 +5385,63 @@ static synchronized long getColumnEncryptionKeyCacheTtl() { * Used to keep track of an individual prepared statement handle on the server side. */ static class PreparedStatementHandle { - private final AtomicInteger handle; + private int handle; private final AtomicInteger handleRefCount = new AtomicInteger(); private boolean isDirectSql; PreparedStatementHandle() { - handle = new AtomicInteger(); + handle = 0; } PreparedStatementHandle(int handle, boolean isDirectSql) { - this.handle = new AtomicInteger(handle); + this.handle = handle; this.isDirectSql = isDirectSql; } int getHandle() { - return handle.get(); + return handle; } - void setHandle(int handle, boolean isDirectSql) { + boolean setHandle(int handle, boolean isDirectSql) { if (handleRefCount.compareAndSet(0, 1)) { - this.handle.compareAndSet(0, handle); + this.handle = handle; this.isDirectSql = isDirectSql; + return true; } + else + return false; } public boolean isDirectSql() { return isDirectSql; } - public int getRefCount() { - return handleRefCount.get(); + public boolean killHandle() { + if(!hasHandle()) + return false; + else + return handleRefCount.compareAndSet(0, -999); } - public void incrementRefCount() { - this.handleRefCount.incrementAndGet(); + public boolean isKilled() { + return 0 > handleRefCount.intValue(); } - public int decrementRefCount() { - return this.handleRefCount.decrementAndGet(); + boolean hasHandle() { + return 0 < getHandle(); + } + + public boolean addReference() { + if (!hasHandle() || isKilled()) + return false; + else { + int refCount = handleRefCount.incrementAndGet(); + return refCount > 0; + } + } + + public void removeReference() { + handleRefCount.decrementAndGet(); } } @@ -5640,10 +5646,7 @@ final void handlePreparedStatementDiscardActions(boolean force) { PreparedStatementHandle statementHandle = null; while (null != (statementHandle = discardedPreparedStatementHandles.poll())){ - if (0 < statementHandle.getRefCount()) - continue; // Will get discarded when remaining prepared statements get closed. - - ++handlesRemoved; + ++handlesRemoved; sql.append(statementHandle.isDirectSql() ? "EXEC sp_unprepare " : "EXEC sp_cursorunprepare ") .append(statementHandle.getHandle()) @@ -5728,8 +5731,12 @@ final PreparedStatementCacheItem borrowCachedPreparedStatementMetadata(Sha1HashK /** Return prepared statement cache entry so it can be un-prepared. */ final void returnCachedPreparedStatementMetadata(PreparedStatementCacheItem cacheItem) { - if (0 >= cacheItem.decrementHandleRefCount() && cacheItem.evictedFromCache && cacheItem.hasHandle()) - enqueueUnprepareStatementHandle(cacheItem.statementHandle); + if(cacheItem.hasHandle()) { + cacheItem.statementHandle.removeReference(); + + if (cacheItem.evictedFromCache && cacheItem.statementHandle.killHandle()) + enqueueUnprepareStatementHandle(cacheItem.statementHandle); + } } // Handle closing handles when removed from cache. @@ -5739,7 +5746,7 @@ public void onEviction(Sha1HashKey key, PreparedStatementCacheItem cacheItem) { cacheItem.evictedFromCache = true; // Mark as evicted from cache. // Only discard if not referenced. - if(cacheItem.hasHandle() && 0 >= cacheItem.getHandleRefCount()) { + if(cacheItem.hasHandle() && cacheItem.statementHandle.killHandle()) { enqueueUnprepareStatementHandle(cacheItem.statementHandle); // Do not run discard actions here! Can interfere with executing statement. } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 638f0cdaf..404f310f5 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -70,7 +70,7 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS private boolean isExecutedAtLeastOnce = false; /** Reference to cache item for statement pooling. Only used to decrement ref count on statement close. */ - private final PreparedStatementCacheItem cachedPreparedStatementItem; + private PreparedStatementCacheItem cachedPreparedStatementItem; /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set @@ -180,9 +180,10 @@ String getClassNameInternal() { cachedPreparedStatementItem = connection.borrowCachedPreparedStatementMetadata(cacheKey); // If handle was found then re-use. if(null != cachedPreparedStatementItem && cachedPreparedStatementItem.hasExecutedSpExecuteSql()) { - // If existing handle was found use it. - if(cachedPreparedStatementItem.hasHandle()) { - prepStmtHandle = cachedPreparedStatementItem.getHandleAndIncrementRefCount(); + // If existing handle was found and we can add reference use it. + PreparedStatementHandle handle = cachedPreparedStatementItem.getPreparedStatementHandle(); + if (handle.addReference()) { + prepStmtHandle = handle.getHandle(); } else { // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. @@ -548,8 +549,12 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { param.skipRetValStatus(tdsReader); prepStmtHandle = param.getInt(tdsReader); - if (null != cachedPreparedStatementItem) - cachedPreparedStatementItem.setHandle(prepStmtHandle, executedSqlDirectly); + if(null != cachedPreparedStatementItem + && !cachedPreparedStatementItem + .getPreparedStatementHandle() + .setHandle(prepStmtHandle, executedSqlDirectly) + ) + cachedPreparedStatementItem = null; // Handle could not be set, treat as not cached. param.skipValue(tdsReader, true); if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) @@ -885,7 +890,10 @@ private boolean doPrepExec(TDSWriter tdsWriter, else { // Move overhead of needing to do prepare & unprepare to only use cases that need more than one execution. // First execution, use sp_executesql, optimizing for asumption we will not re-use statement. - if (needsPrepare && !connection.getEnablePrepareOnFirstPreparedStatementCall() && !isExecutedAtLeastOnce) { + if (needsPrepare + && !connection.getEnablePrepareOnFirstPreparedStatementCall() + && !isExecutedAtLeastOnce + ) { buildExecSQLParams(tdsWriter); isExecutedAtLeastOnce = true; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 46700a671..56bb4d424 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -163,7 +163,6 @@ public void testStatementPooling() throws SQLException { int handle = 0; try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { pstmt.execute(); // sp_prepexec - pstmt.getMoreResults(); // Make sure handle is updated. handle = pstmt.getPreparedStatementHandle(); From 86e15f2473c99356f2892cab656e8bbde8d5f808 Mon Sep 17 00:00:00 2001 From: tobiast Date: Mon, 15 May 2017 07:57:37 -0700 Subject: [PATCH 18/34] Updated comments + race condition test from Brett --- .../jdbc/SQLServerPreparedStatement.java | 4 +- .../unit/statement/PreparedStatementTest.java | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 638f0cdaf..7f43192df 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -214,12 +214,12 @@ private void closePreparedHandle() { if (null != cachedPreparedStatementItem) { connection.returnCachedPreparedStatementMetadata(cachedPreparedStatementItem); } - // Using batched clean-up? If not, use old method of calling sp_unprepare. + // Using batched clean-up? else if(1 < connection.getServerPreparedStatementDiscardThreshold()) { connection.enqueueUnprepareStatementHandle(new PreparedStatementHandle(getPreparedStatementHandle(), executedSqlDirectly)); } else { - // Non batched behavior (same as pre batch impl.) + // Non batched behavior (same as pre batch clean-up implementation) if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) loggerExternal.finer(this + ": Closing PreparedHandle:" + handleToClose); diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 46700a671..e69e00a79 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -7,9 +7,11 @@ */ package com.microsoft.sqlserver.jdbc.unit.statement; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; import java.sql.DriverManager; @@ -17,6 +19,9 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.platform.runner.JUnitPlatform; @@ -283,6 +288,54 @@ public void testStatementPoolingEviction() throws SQLException { } } + + @Test + public void testPrepareRace() throws Exception { + // Make sure correct settings are used. + SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(true); + SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(2); + + String[] queries = new String[3]; + queries[0] = String.format("SELECT * FROM sys.tables -- %s", UUID.randomUUID()); + queries[1] = String.format("SELECT * FROM sys.tables -- %s", UUID.randomUUID()); + queries[2] = String.format("SELECT * FROM sys.tables -- %s", UUID.randomUUID()); + + ExecutorService threadPool = Executors.newFixedThreadPool(4); + AtomicReference exception = new AtomicReference<>(); + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + + for (int i = 0; i < 4; i++) { + threadPool.execute(() -> { + for (int j = 0; j < 500000; j++) { + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement(queries[j % 3])) { + pstmt.execute(); + } + catch (SQLException e) { + exception.set(e); + break; + } + } + }); + } + + threadPool.shutdown(); + threadPool.awaitTermination(12000, SECONDS); + + assertNull(exception.get()); + + // Force un-prepares. + con.closeDiscardedServerPreparedStatements(); + + // Verify that queue is now empty. + assertSame(0, con.getDiscardedServerPreparedStatementCount()); + + } + finally { + SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); + SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); + } + } + /** * Test handling of the two configuration knobs related to prepared statement handling. * From 3b1159cb50ab35cfaf372c2767b61688d85da2f1 Mon Sep 17 00:00:00 2001 From: tobiast Date: Tue, 16 May 2017 09:58:45 -0700 Subject: [PATCH 19/34] Fixed issues related to cursors, clean-up. --- ...LCacheItem.java => ParsedSQLMetadata.java} | 4 +- .../sqlserver/jdbc/SQLServerConnection.java | 452 ++++++++---------- .../sqlserver/jdbc/SQLServerDataSource.java | 8 +- .../sqlserver/jdbc/SQLServerDriver.java | 10 +- .../jdbc/SQLServerPreparedStatement.java | 119 +++-- .../sqlserver/jdbc/SQLServerStatement.java | 2 +- .../unit/statement/PreparedStatementTest.java | 79 +-- 7 files changed, 307 insertions(+), 367 deletions(-) rename src/main/java/com/microsoft/sqlserver/jdbc/{ParsedSQLCacheItem.java => ParsedSQLMetadata.java} (82%) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java similarity index 82% rename from src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java rename to src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java index 19c34ebec..f047ff71a 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLCacheItem.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java @@ -11,14 +11,14 @@ /** * Used for caching of meta data from parsed SQL text. */ -final class ParsedSQLCacheItem { +final class ParsedSQLMetadata { /** The SQL text AFTER processing. */ String processedSQL; int parameterCount; String procedureName; boolean bReturnValueSyntax; - ParsedSQLCacheItem(String processedSQL, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + ParsedSQLMetadata(String processedSQL, int parameterCount, String procedureName, boolean bReturnValueSyntax) { this.processedSQL = processedSQL; this.parameterCount = parameterCount; this.procedureName = procedureName; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index ef7548911..427610d84 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -89,17 +89,15 @@ public class SQLServerConnection implements ISQLServerConnection { // Threasholds related to when prepared statement handles are cleaned-up. 1 == immediately. /** - * The initial default on application start-up for the prepared statement clean-up action threshold (i.e. when sp_unprepare is called). + * The default for the prepared statement clean-up action threshold (i.e. when sp_unprepare is called). */ - static final private int INITIAL_DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD = 10; // Used to set the initial default, can be changed later. - static private int defaultServerPreparedStatementDiscardThreshold = -1; // Current default for new connections + static final int DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD = 10; // Used to set the initial default, can be changed later. private int serverPreparedStatementDiscardThreshold = -1; // Current limit for this particular connection. /** - * The initial default on application start-up for if prepared statements should execute sp_executesql before following the prepare, unprepare pattern. + * The default for if prepared statements should execute sp_executesql before following the prepare, unprepare pattern. */ - static final private boolean INITIAL_DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL = false; // Used to set the initial default, can be changed later. false == use sp_executesql -> sp_prepexec -> sp_execute -> batched -> sp_unprepare pattern, true == skip sp_executesql part of pattern. - static private Boolean defaultEnablePrepareOnFirstPreparedStatementCall = null; // Current default for new connections + static final boolean DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL = false; // Used to set the initial default, can be changed later. false == use sp_executesql -> sp_prepexec -> sp_execute -> batched -> sp_unprepare pattern, true == skip sp_executesql part of pattern. private Boolean enablePrepareOnFirstPreparedStatementCall = null; // Current limit for this particular connection. // Handle the actual queue of discarded prepared statements. @@ -148,21 +146,130 @@ private java.security.MessageDigest getSha1Digest() { } } + /** + * Used to keep track of an individual prepared statement handle. + */ + static class PreparedStatementHandle { + private int handle = 0; + private final AtomicInteger handleRefCount = new AtomicInteger(); + private boolean isDirectSql; + private volatile boolean hasExecutedAtLeastOnce; + private volatile boolean evictedFromCache; + + PreparedStatementHandle() { + } + + /** Has the statement been evicted from the statement handle cache. */ + private boolean isEvictedFromCache() { + return evictedFromCache; + } + + /** Specify whether the statement been evicted from the statement handle cache. */ + private void setIsEvictedFromCache(boolean isEvictedFromCache) { + this.evictedFromCache = isEvictedFromCache; + } + + /** Has the statement that this instance is related to ever been executed (with or without handle) */ + boolean hasExecutedAtLeastOnce() { + return hasExecutedAtLeastOnce; + } + + /** Specify whether the statement that this instance is related to ever been executed (with or without handle) */ + void setHasExecutedAtLeastOnce(boolean hasExecutedAtLeastOnce) { + this.hasExecutedAtLeastOnce = hasExecutedAtLeastOnce; + } + + PreparedStatementHandle(int handle, boolean isDirectSql, boolean isEvictedFromCache) { + this.handle = handle; + this.isDirectSql = isDirectSql; + this.setHasExecutedAtLeastOnce(true); + this.setIsEvictedFromCache(isEvictedFromCache); + } + + /** Get the actual handle. */ + int getHandle() { + return handle; + } + + /** Specify the handle. + * + * @return + * false: Handle could not be referenced (already references other handle). + * true: Handle was successfully set. + */ + boolean setHandle(int handle, boolean isDirectSql) { + if (handleRefCount.compareAndSet(0, 1)) { + this.handle = handle; + this.isDirectSql = isDirectSql; + return true; + } + else + return false; + } + + boolean isDirectSql() { + return isDirectSql; + } + + /** Make sure handle cannot be re-used. + * + * @return + * false: Handle could not be discarded, it is in use. + * true: Handle was successfully put on path for discarding. + */ + private boolean tryDiscardHandle() { + if(!hasHandle()) + return false; + else + return handleRefCount.compareAndSet(0, -999); + } + + /** Returns whether this statement has been discarded and can no longer be re-used. */ + private boolean isDiscarded() { + return 0 > handleRefCount.intValue(); + } + + /** Returns whether this statement has an actual server handle associated with it. */ + private boolean hasHandle() { + return 0 < getHandle(); + } + + /** Adds a new reference to this handle, i.e. re-using it. + * + * @return + * false: Reference could not be added, statement has been discarded or does not have a handle associated with it. + * true: Reference was successfully added. + */ + boolean tryAddReference() { + if (!hasHandle() || isDiscarded()) + return false; + else { + int refCount = handleRefCount.incrementAndGet(); + return refCount > 0; + } + } + + /** Remove a reference from this handle*/ + private void removeReference() { + handleRefCount.decrementAndGet(); + } + } + /** Size of the parsed SQL-text metadata cache */ static final private int PARSED_SQL_CACHE_SIZE = 100; /** Cache of parsed SQL meta data */ - static private ConcurrentLinkedHashMap parsedSQLCache; + static private ConcurrentLinkedHashMap parsedSQLCache; static { - parsedSQLCache = new Builder() + parsedSQLCache = new Builder() .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE) .build(); } /** Get prepared statement cache entry if exists, if not parse and create a new one */ - static ParsedSQLCacheItem getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, String sql) throws SQLServerException { - ParsedSQLCacheItem cacheItem = parsedSQLCache.get(key); + static ParsedSQLMetadata getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, String sql) throws SQLServerException { + ParsedSQLMetadata cacheItem = parsedSQLCache.get(key); if (null == cacheItem) { JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); @@ -171,13 +278,23 @@ static ParsedSQLCacheItem getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, St boolean returnValueSyntax = translator.hasReturnValueSyntax(); int paramCount = countParams(parsedSql); - cacheItem = new ParsedSQLCacheItem(parsedSql, paramCount, procName, returnValueSyntax); + cacheItem = new ParsedSQLMetadata(parsedSql, paramCount, procName, returnValueSyntax); parsedSQLCache.putIfAbsent(key, cacheItem); } return cacheItem; } + /** Size of the prepared statement handle cache */ + private int statementPoolingCacheSize = 10; + + /** Default size for prepared statement caches */ + static final int DEFAULT_STATEMENT_POOLING_CACHE_SIZE = 10; + /** Cache of prepared statement handles */ + private ConcurrentLinkedHashMap preparedStatementHandleCache; + /** Cache of prepared statement parameter metadata */ + private ConcurrentLinkedHashMap parameterMetadataCache; + /** * Find statement parameters. * @@ -196,50 +313,6 @@ private static int countParams(String sql) { return nParams; } - /** - * Used to keep track of an individual handle ready for un-prepare. - */ - final class PreparedStatementCacheItem { - private final PreparedStatementHandle statementHandle = new PreparedStatementHandle(); - private volatile SQLServerParameterMetaData parameterMetadata; - private volatile boolean hasExecutedSpExecuteSql; - volatile boolean evictedFromCache; - - boolean hasExecutedSpExecuteSql() { - return hasExecutedSpExecuteSql; - } - - void setHasExecutedSpExecuteSql(boolean hasExecutedSpExecuteSql) { - this.hasExecutedSpExecuteSql = hasExecutedSpExecuteSql; - } - - boolean hasHandle() { - return 0 < statementHandle.getHandle(); - } - - PreparedStatementHandle getPreparedStatementHandle() { - return statementHandle; - } - - boolean hasParameterMetadata() { - return null != this.parameterMetadata; - } - - SQLServerParameterMetaData getParameterMetadata() { - return parameterMetadata; - } - - void setParameterMetadata(SQLServerParameterMetaData metadata) { - parameterMetadata = metadata; - } - } - - /** Size of the prepared statement handle cache */ - private int statementPoolingCacheSize = 10; - - /** Cache of prepared statement handles */ - private ConcurrentLinkedHashMap preparedStatementCache; - SqlFedAuthToken getAuthenticationResult() { return fedAuthToken; } @@ -846,12 +919,16 @@ final boolean attachConnId() { throw new UnsupportedOperationException(message); } - // Caching turned off? - if (isStatementPoolingEnabled()) { - preparedStatementCache = new Builder() - .maximumWeightedCapacity(getStatementPoolingCacheSize()) - .listener(new PreparedStatementCacheEvictionListener()) - .build(); + // Caching turned on? + if (0 < this.getStatementPoolingCacheSize()) { + preparedStatementHandleCache = new Builder() + .maximumWeightedCapacity(getStatementPoolingCacheSize()) + .listener(new PreparedStatementCacheEvictionListener()) + .build(); + + parameterMetadataCache = new Builder() + .maximumWeightedCapacity(getStatementPoolingCacheSize()) + .build(); } } @@ -2846,9 +2923,12 @@ public void close() throws SQLServerException { tdsChannel.close(); } - // Invalidate statement cache. - if(null != this.preparedStatementCache) - this.preparedStatementCache.clear(); + // Invalidate statement caches. + if(null != preparedStatementHandleCache) + preparedStatementHandleCache.clear(); + + if(null != parameterMetadataCache) + parameterMetadataCache.clear(); // Clean-up queue etc. related to batching of prepared statement discard actions (sp_unprepare). cleanupPreparedStatementDiscardActions(); @@ -5379,72 +5459,7 @@ public static synchronized void setColumnEncryptionKeyCacheTtl(int columnEncrypt static synchronized long getColumnEncryptionKeyCacheTtl() { return columnEncryptionKeyCacheTtl; } - - /** - * Used to keep track of an individual prepared statement handle on the server side. - */ - static class PreparedStatementHandle { - private int handle; - private final AtomicInteger handleRefCount = new AtomicInteger(); - private boolean isDirectSql; - - PreparedStatementHandle() { - handle = 0; - } - - PreparedStatementHandle(int handle, boolean isDirectSql) { - this.handle = handle; - this.isDirectSql = isDirectSql; - } - - int getHandle() { - return handle; - } - - boolean setHandle(int handle, boolean isDirectSql) { - if (handleRefCount.compareAndSet(0, 1)) { - this.handle = handle; - this.isDirectSql = isDirectSql; - return true; - } - else - return false; - } - - public boolean isDirectSql() { - return isDirectSql; - } - - public boolean killHandle() { - if(!hasHandle()) - return false; - else - return handleRefCount.compareAndSet(0, -999); - } - - public boolean isKilled() { - return 0 > handleRefCount.intValue(); - } - - boolean hasHandle() { - return 0 < getHandle(); - } - - public boolean addReference() { - if (!hasHandle() || isKilled()) - return false; - else { - int refCount = handleRefCount.incrementAndGet(); - return refCount > 0; - } - } - - public void removeReference() { - handleRefCount.decrementAndGet(); - } - } - /** * Enqueue a discarded prepared statement handle to be clean-up on the server. @@ -5453,12 +5468,17 @@ public void removeReference() { * The prepared statement handle that should be scheduled for unprepare. */ final void enqueueUnprepareStatementHandle(PreparedStatementHandle statementHandle) { + if(null == statementHandle) + return; + if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) loggerExternal.finer(this + ": Adding PreparedHandle to queue for un-prepare:" + statementHandle.getHandle()); // Add the new handle to the discarding queue and find out current # enqueued. - this.discardedPreparedStatementHandles.add(statementHandle); - this.discardedPreparedStatementHandleCount.incrementAndGet(); + if(statementHandle.hasHandle()) { + this.discardedPreparedStatementHandles.add(statementHandle); + this.discardedPreparedStatementHandleCount.incrementAndGet(); + } } @@ -5474,53 +5494,16 @@ public int getDiscardedServerPreparedStatementCount() { /** * Forces the un-prepare requests for any outstanding discarded prepared statements to be executed. */ - public void closeDiscardedServerPreparedStatements() { - this.handlePreparedStatementDiscardActions(true); + public void closeUnreferencedPreparedStatementHandles() { + this.unprepareUnreferencedPreparedStatementHandles(true); } /** * Remove references to outstanding un-prepare requests. Should be run when connection is closed. */ private final void cleanupPreparedStatementDiscardActions() { - this.discardedPreparedStatementHandles.clear(); - this.discardedPreparedStatementHandleCount.set(0); - } - - /** - * The initial default on application start-up for if prepared statements should execute sp_executesql before following the prepare, unprepare pattern. - * - * @return Returns the current setting per the description. - */ - static public boolean getInitialDefaultEnablePrepareOnFirstPreparedStatementCall() { - return INITIAL_DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL; - } - - /** - * Returns the default behavior for new connection instances. If false the first execution will call sp_executesql and not prepare - * a statement, once the second execution happens it will call sp_prepexec and actually setup a prepared statement handle. Following - * executions will call sp_execute. This relieves the need for sp_unprepare on prepared statement close if the statement is only - * executed once. Initial setting for this option is available in INITIAL_DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL. - * - * @return Returns the current setting per the description. - */ - static public boolean getDefaultEnablePrepareOnFirstPreparedStatementCall() { - if(null == defaultEnablePrepareOnFirstPreparedStatementCall) - return getInitialDefaultEnablePrepareOnFirstPreparedStatementCall(); - else - return defaultEnablePrepareOnFirstPreparedStatementCall; - } - - /** - * Specifies the default behavior for new connection instances. If value is false the first execution will call sp_executesql and not prepare - * a statement, once the second execution happens it will call sp_prepexec and actually setup a prepared statement handle. Following - * executions will call sp_execute. This relieves the need for sp_unprepare on prepared statement close if the statement is only - * executed once. Initial setting for this option is available in INITIAL_DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL. - * - * @param value - * Changes the setting per the description. - */ - static public void setDefaultEnablePrepareOnFirstPreparedStatementCall(boolean value) { - defaultEnablePrepareOnFirstPreparedStatementCall = value; + discardedPreparedStatementHandles.clear(); + discardedPreparedStatementHandleCount.set(0); } /** @@ -5533,7 +5516,7 @@ static public void setDefaultEnablePrepareOnFirstPreparedStatementCall(boolean v */ public boolean getEnablePrepareOnFirstPreparedStatementCall() { if(null == this.enablePrepareOnFirstPreparedStatementCall) - return getDefaultEnablePrepareOnFirstPreparedStatementCall(); + return DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL; else return this.enablePrepareOnFirstPreparedStatementCall; } @@ -5551,45 +5534,6 @@ public void setEnablePrepareOnFirstPreparedStatementCall(boolean value) { this.enablePrepareOnFirstPreparedStatementCall = value; } - /** - * The initial default on application start-up for the prepared statement clean-up action threshold (i.e. when sp_unprepare is called). - * - * @return Returns the current setting per the description. - */ - static public int getInitialDefaultServerPreparedStatementDiscardThreshold() { - return INITIAL_DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD; - } - - /** - * Returns the default behavior for new connection instances. This setting controls how many outstanding prepared statement discard - * actions (sp_unprepare) can be outstanding per connection before a call to clean-up the outstanding handles on the server is executed. - * If the setting is <= 1 unprepare actions will be executed immedietely on prepared statement close. If it is set to >1 these calls will - * be batched together to avoid overhead of calling sp_unprepare too often. - * Initial setting for this option is available in INITIAL_DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD. - * - * @return Returns the current setting per the description. - */ - static public int getDefaultServerPreparedStatementDiscardThreshold() { - if (0 > defaultServerPreparedStatementDiscardThreshold) - return getInitialDefaultServerPreparedStatementDiscardThreshold(); - else - return defaultServerPreparedStatementDiscardThreshold; - } - - /** - * Specifies the default behavior for new connection instances. This setting controls how many outstanding prepared statement discard - * actions (sp_unprepare) can be outstanding per connection before a call to clean-up the outstanding handles on the server is executed. - * If the setting is <= 1 unprepare actions will be executed immedietely on prepared statement close. If it is set to >1 these calls will - * be batched together to avoid overhead of calling sp_unprepare too often. - * Initial setting for this option is available in INITIAL_DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD. - * - * @param value - * Changes the setting per the description. - */ - static public void setDefaultServerPreparedStatementDiscardThreshold(int value) { - defaultServerPreparedStatementDiscardThreshold = Math.max(0, value); - } - /** * Returns the behavior for a specific connection instance. This setting controls how many outstanding prepared statement discard * actions (sp_unprepare) can be outstanding per connection before a call to clean-up the outstanding handles on the server is executed. @@ -5601,9 +5545,9 @@ static public void setDefaultServerPreparedStatementDiscardThreshold(int value) */ public int getServerPreparedStatementDiscardThreshold() { if (0 > this.serverPreparedStatementDiscardThreshold) - return getDefaultServerPreparedStatementDiscardThreshold(); + return DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD; else - return this.serverPreparedStatementDiscardThreshold; + return this.serverPreparedStatementDiscardThreshold; } /** @@ -5619,13 +5563,17 @@ public void setServerPreparedStatementDiscardThreshold(int value) { this.serverPreparedStatementDiscardThreshold = Math.max(0, value); } + final boolean isPreparedStatementUnprepareBatchingEnabled() { + return 1 < getServerPreparedStatementDiscardThreshold(); + } + /** * Cleans-up discarded prepared statement handles on the server using batched un-prepare actions if the batching threshold has been reached. * * @param force * When force is set to true we ignore the current threshold for if the discard actions should run and run them anyway. */ - final void handlePreparedStatementDiscardActions(boolean force) { + final void unprepareUnreferencedPreparedStatementHandles(boolean force) { // Skip out if session is unavailable to adhere to previous non-batched behavior. if (isSessionUnAvailable()) return; @@ -5668,7 +5616,7 @@ final void handlePreparedStatementDiscardActions(boolean force) { } // Decrement threshold counter - this.discardedPreparedStatementHandleCount.addAndGet(-handlesRemoved); + discardedPreparedStatementHandleCount.addAndGet(-handlesRemoved); } } @@ -5678,18 +5626,18 @@ final void handlePreparedStatementDiscardActions(boolean force) { * @return Returns the current setting per the description. */ public int getStatementPoolingCacheSize() { - return this.statementPoolingCacheSize; + return statementPoolingCacheSize; } /** - * Returns the current number of pooled prepared statements. + * Returns the current number of pooled prepared statement handles. * @return Returns the current setting per the description. */ - public int getStatementPoolingCacheEntryCount() { - if(null == this.preparedStatementCache) + public int getStatementHandleCacheEntryCount() { + if(!isStatementPoolingEnabled()) return 0; else - return this.preparedStatementCache.size(); + return this.preparedStatementHandleCache.size(); } /** @@ -5697,7 +5645,7 @@ public int getStatementPoolingCacheEntryCount() { * @return Returns the current setting per the description. */ public boolean isStatementPoolingEnabled() { - return 0 < this.getStatementPoolingCacheSize(); + return null != preparedStatementHandleCache && 0 < this.getStatementPoolingCacheSize(); } /** @@ -5707,47 +5655,65 @@ public boolean isStatementPoolingEnabled() { public void setStatementPoolingCacheSize(int value) { if (value != this.statementPoolingCacheSize) { value = Math.max(0, value); - this.statementPoolingCacheSize = value; + statementPoolingCacheSize = value; - if (null != this.preparedStatementCache) { - this.preparedStatementCache.setCapacity(value); - } + if (null != preparedStatementHandleCache) + preparedStatementHandleCache.setCapacity(value); + + if (null != parameterMetadataCache) + parameterMetadataCache.setCapacity(value); } } - /** Get or create prepared statement cache entry if statement pooling is enabled */ - final PreparedStatementCacheItem borrowCachedPreparedStatementMetadata(Sha1HashKey key) { + /** Get a parameter metadata cache entry if statement pooling is enabled */ + final SQLServerParameterMetaData getCachedParameterMetadata(Sha1HashKey key) { + if(!isStatementPoolingEnabled()) + return null; + + return parameterMetadataCache.get(key); + } + + /** Register a parameter metadata cache entry if statement pooling is enabled */ + final void registerCachedParameterMetadata(Sha1HashKey key, SQLServerParameterMetaData pmd) { + if(!isStatementPoolingEnabled() || null == pmd) + return; + + parameterMetadataCache.put(key, pmd); + } + + /** Get or create prepared statement handle cache entry if statement pooling is enabled */ + final PreparedStatementHandle getOrRegisterCachedPreparedStatementHandle(Sha1HashKey key) { if(!isStatementPoolingEnabled()) return null; - PreparedStatementCacheItem cacheItem = preparedStatementCache.get(key); + PreparedStatementHandle cacheItem = preparedStatementHandleCache.get(key); if (null == cacheItem) { - cacheItem = new PreparedStatementCacheItem(); - preparedStatementCache.putIfAbsent(key, cacheItem); + cacheItem = new PreparedStatementHandle(); + preparedStatementHandleCache.putIfAbsent(key, cacheItem); } return cacheItem; } - /** Return prepared statement cache entry so it can be un-prepared. */ - final void returnCachedPreparedStatementMetadata(PreparedStatementCacheItem cacheItem) { - if(cacheItem.hasHandle()) { - cacheItem.statementHandle.removeReference(); + /** Return prepared statement handle cache entry so it can be un-prepared. */ + final void returnCachedPreparedStatementHandle(PreparedStatementHandle handle) { + if(handle.hasHandle()) { + handle.removeReference(); - if (cacheItem.evictedFromCache && cacheItem.statementHandle.killHandle()) - enqueueUnprepareStatementHandle(cacheItem.statementHandle); + if (handle.isEvictedFromCache() && handle.tryDiscardHandle()) + enqueueUnprepareStatementHandle(handle); } } // Handle closing handles when removed from cache. - final class PreparedStatementCacheEvictionListener implements EvictionListener { - public void onEviction(Sha1HashKey key, PreparedStatementCacheItem cacheItem) { - if(null != cacheItem) { - cacheItem.evictedFromCache = true; // Mark as evicted from cache. + final class PreparedStatementCacheEvictionListener implements EvictionListener { + public void onEviction(Sha1HashKey key, PreparedStatementHandle handle) { + if(null != handle) { + handle.setIsEvictedFromCache(true); // Mark as evicted from cache. // Only discard if not referenced. - if(cacheItem.hasHandle() && cacheItem.statementHandle.killHandle()) { - enqueueUnprepareStatementHandle(cacheItem.statementHandle); + if(handle.hasHandle() && handle.tryDiscardHandle()) { + enqueueUnprepareStatementHandle(handle); // Do not run discard actions here! Can interfere with executing statement. } } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java index 80ee1e1e6..6cd76fdf1 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java @@ -683,8 +683,8 @@ public void setEnablePrepareOnFirstPreparedStatementCall(boolean enablePrepareOn * @return Returns the current setting per the description. */ public boolean getEnablePrepareOnFirstPreparedStatementCall() { - return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), - SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); + boolean defaultValue = SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.getDefaultValue(); + return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), defaultValue); } /** @@ -709,8 +709,8 @@ public void setServerPreparedStatementDiscardThreshold(int serverPreparedStateme * @return Returns the current setting per the description. */ public int getServerPreparedStatementDiscardThreshold() { - return getIntProperty(connectionProps, SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.toString(), - SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); + int defaultSize = SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.getDefaultValue(); + return getIntProperty(connectionProps, SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.toString(), defaultSize); } /** diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java index 84f229be1..1ea9f3125 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDriver.java @@ -274,8 +274,8 @@ enum SQLServerDriverIntProperty { QUERY_TIMEOUT ("queryTimeout", -1), PORT_NUMBER ("portNumber", 1433), SOCKET_TIMEOUT ("socketTimeout", 0), - SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD("serverPreparedStatementDiscardThreshold", -1/*This is not the default, default handled in SQLServerConnection and is not final/const*/), - STATEMENT_POOLING_CACHE_SIZE ("statementPoolingCacheSize", 10), + SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD("serverPreparedStatementDiscardThreshold", SQLServerConnection.DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD), + STATEMENT_POOLING_CACHE_SIZE ("statementPoolingCacheSize", SQLServerConnection.DEFAULT_STATEMENT_POOLING_CACHE_SIZE), ; private String name; @@ -310,7 +310,7 @@ enum SQLServerDriverBooleanProperty TRUST_SERVER_CERTIFICATE ("trustServerCertificate", false), XOPEN_STATES ("xopenStates", false), FIPS ("fips", false), - ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT("enablePrepareOnFirstPreparedStatementCall", false/*This is not the default, default handled in SQLServerConnection and is not final/const*/); + ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT("enablePrepareOnFirstPreparedStatementCall", SQLServerConnection.DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL); private String name; private boolean defaultValue; @@ -380,8 +380,8 @@ public final class SQLServerDriver implements java.sql.Driver { new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.FIPS_PROVIDER.toString(), SQLServerDriverStringProperty.FIPS_PROVIDER.getDefaultValue(), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.SOCKET_TIMEOUT.toString(), Integer.toString(SQLServerDriverIntProperty.SOCKET_TIMEOUT.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.FIPS.toString(), Boolean.toString(SQLServerDriverBooleanProperty.FIPS.getDefaultValue()), false, TRUE_FALSE), - new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), Boolean.toString(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()), false, TRUE_FALSE), - new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.toString(), Integer.toString(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()), false, null), + new SQLServerDriverPropertyInfo(SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), Boolean.toString(SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.getDefaultValue()), false,TRUE_FALSE), + new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.toString(), Integer.toString(SQLServerDriverIntProperty.SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.toString(), Integer.toString(SQLServerDriverIntProperty.STATEMENT_POOLING_CACHE_SIZE.getDefaultValue()), false, null), new SQLServerDriverPropertyInfo(SQLServerDriverStringProperty.JAAS_CONFIG_NAME.toString(), SQLServerDriverStringProperty.JAAS_CONFIG_NAME.getDefaultValue(), false, null), }; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 1fe6696b3..5aab030a5 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -30,7 +30,6 @@ import java.util.Vector; import java.util.logging.Level; -import com.microsoft.sqlserver.jdbc.SQLServerConnection.PreparedStatementCacheItem; import com.microsoft.sqlserver.jdbc.SQLServerConnection.PreparedStatementHandle; import com.microsoft.sqlserver.jdbc.SQLServerConnection.Sha1HashKey; @@ -69,8 +68,11 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** True if this execute has been called for this statement at least once */ private boolean isExecutedAtLeastOnce = false; - /** Reference to cache item for statement pooling. Only used to decrement ref count on statement close. */ - private PreparedStatementCacheItem cachedPreparedStatementItem; + /** Reference to cache item for statement handle pooling. Only used to decrement ref count on statement close. */ + private PreparedStatementHandle cachedPreparedStatementHandle; + + /** Hash of user supplied SQL statement used for various cache lookups */ + private Sha1HashKey cacheKey; /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set @@ -95,6 +97,20 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** The prepared statement handle returned by the server */ private int prepStmtHandle = 0; + /** Whether the prepared statement handle is used for cursors or not (then default result set) */ + private boolean isPrepStmtHandleCursorable = false; + + private boolean isPrepStmtHandleCursorable() { + return isPrepStmtHandleCursorable; + } + + private void setIsPrepStmtHandleCursorable(boolean value) { + this.isPrepStmtHandleCursorable = value; + } + + private void setPreparedStatementHandle(int handle) { + this.prepStmtHandle = handle; + } /** The server handle for this prepared statement. If a value < 1 is returned no handle has been created. * @@ -118,6 +134,7 @@ private boolean hasPreparedStatementHandle() { */ private void resetPrepStmtHandle() { prepStmtHandle = 0; + setIsPrepStmtHandleCursorable(false); } /** Flag set to true when statement execution is expected to return the prepared statement handle */ @@ -165,29 +182,28 @@ String getClassNameInternal() { stmtPoolable = true; // Create a cache key for this statement. - Sha1HashKey cacheKey = new Sha1HashKey(sql); + cacheKey = new Sha1HashKey(sql); // Parse or fetch SQL metadata from cache. - ParsedSQLCacheItem cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); + ParsedSQLMetadata parsedSQL = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); // Retrieve meta data from cache item. - procedureName = cacheItem.procedureName; - bReturnValueSyntax = cacheItem.bReturnValueSyntax; - userSQL = cacheItem.processedSQL; - initParams(cacheItem.parameterCount); + procedureName = parsedSQL.procedureName; + bReturnValueSyntax = parsedSQL.bReturnValueSyntax; + userSQL = parsedSQL.processedSQL; + initParams(parsedSQL.parameterCount); // See if existing prepared statement handle can be re-used. - cachedPreparedStatementItem = connection.borrowCachedPreparedStatementMetadata(cacheKey); + cachedPreparedStatementHandle = connection.getOrRegisterCachedPreparedStatementHandle(cacheKey); // If handle was found then re-use. - if(null != cachedPreparedStatementItem && cachedPreparedStatementItem.hasExecutedSpExecuteSql()) { - // If existing handle was found and we can add reference use it. - PreparedStatementHandle handle = cachedPreparedStatementItem.getPreparedStatementHandle(); - if (handle.addReference()) { - prepStmtHandle = handle.getHandle(); - } - else { - // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. - isExecutedAtLeastOnce = true; + if (null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasExecutedAtLeastOnce()) { + // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. + isExecutedAtLeastOnce = true; + + // If existing handle was found and we can add reference to it, use it. + if (cachedPreparedStatementHandle.tryAddReference()) { + setIsPrepStmtHandleCursorable(false); + setPreparedStatementHandle(cachedPreparedStatementHandle.getHandle()); } } } @@ -211,13 +227,13 @@ private void closePreparedHandle() { final int handleToClose = getPreparedStatementHandle(); resetPrepStmtHandle(); - // Handle unprepare actions through batching @ connection level. - if (null != cachedPreparedStatementItem) { - connection.returnCachedPreparedStatementMetadata(cachedPreparedStatementItem); + // Handle unprepare actions through statement pooling. + if (null != cachedPreparedStatementHandle) { + connection.returnCachedPreparedStatementHandle(cachedPreparedStatementHandle); } - // Using batched clean-up? - else if(1 < connection.getServerPreparedStatementDiscardThreshold()) { - connection.enqueueUnprepareStatementHandle(new PreparedStatementHandle(getPreparedStatementHandle(), executedSqlDirectly)); + // If no reference to a statement pool cache item is found handle unprepare actions through batching @ connection level. + else if(connection.isPreparedStatementUnprepareBatchingEnabled()) { + connection.enqueueUnprepareStatementHandle(new PreparedStatementHandle(handleToClose, executedSqlDirectly, true)); } else { // Non batched behavior (same as pre batch clean-up implementation) @@ -255,7 +271,7 @@ final boolean doExecute() throws SQLServerException { } // Always run any outstanding discard actions as statement pooling always uses batched sp_unprepare. - connection.handlePreparedStatementDiscardActions(false); + connection.unprepareUnreferencedPreparedStatementHandles(false); } } @@ -547,14 +563,15 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { expectPrepStmtHandle = false; Parameter param = new Parameter(Util.shouldHonorAEForParameters(stmtColumnEncriptionSetting, connection)); param.skipRetValStatus(tdsReader); - prepStmtHandle = param.getInt(tdsReader); - - if(null != cachedPreparedStatementItem - && !cachedPreparedStatementItem - .getPreparedStatementHandle() - .setHandle(prepStmtHandle, executedSqlDirectly) - ) - cachedPreparedStatementItem = null; // Handle could not be set, treat as not cached. + + setPreparedStatementHandle(param.getInt(tdsReader)); + + // Check if a cache reference should be updated with the newly created handle, NOT for cursorable handles. + if (null != cachedPreparedStatementHandle && !isPrepStmtHandleCursorable()) { + // Attempt to update the handle, if the update fails remove the reference to the cache item since it references a different handle. + if (!cachedPreparedStatementHandle.setHandle(prepStmtHandle, executedSqlDirectly)) + cachedPreparedStatementHandle = null; // Handle could not be set, treat as not cached. + } param.skipValue(tdsReader, true); if (getStatementLogger().isLoggable(java.util.logging.Level.FINER)) @@ -877,11 +894,20 @@ private boolean doPrepExec(TDSWriter tdsWriter, Parameter[] params, boolean hasNewTypeDefinitions, boolean hasExistingTypeDefinitions) throws SQLServerException { - - boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasPreparedStatementHandle(); + + // We only have a handle to re-use if it is both available and same "cursorability". + boolean isCursorableHandle = isCursorable(executeMethod); + + boolean hasHandle = hasPreparedStatementHandle() && isCursorableHandle == isPrepStmtHandleCursorable(); + + boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasHandle; + + // Remember cursorability. + setIsPrepStmtHandleCursorable(isCursorableHandle); - // Cursors never go the non-prepared statement route. - if (isCursorable(executeMethod)) { + // Cursors don't use statement pooling. + if (isCursorableHandle) { + if (needsPrepare) buildServerCursorPrepExecParams(tdsWriter); else @@ -898,8 +924,9 @@ private boolean doPrepExec(TDSWriter tdsWriter, isExecutedAtLeastOnce = true; // Enable re-use if caching is on by moving to sp_prepexec on next call even from separate instance. - if (null != cachedPreparedStatementItem) - cachedPreparedStatementItem.setHasExecutedSpExecuteSql(true); + if (null != cachedPreparedStatementHandle) { + cachedPreparedStatementHandle.setHasExecutedAtLeastOnce(true); + } } // Second execution, use prepared statements since we seem to be re-using it. else if(needsPrepare) @@ -2868,16 +2895,18 @@ public final void setNull(int paramIndex, * Per the description. */ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws SQLServerException { - if (!forceRefresh && null != cachedPreparedStatementItem && cachedPreparedStatementItem.hasParameterMetadata()) { - return cachedPreparedStatementItem.getParameterMetadata(); + + SQLServerParameterMetaData pmd = this.connection.getCachedParameterMetadata(cacheKey); + + if (!forceRefresh && null != pmd) { + return pmd; } else { loggerExternal.entering(getClassNameLogging(), "getParameterMetaData"); checkClosed(); - SQLServerParameterMetaData pmd = new SQLServerParameterMetaData(this, userSQL); + pmd = new SQLServerParameterMetaData(this, userSQL); - if(null != cachedPreparedStatementItem) - cachedPreparedStatementItem.setParameterMetadata(pmd); + connection.registerCachedParameterMetadata(cacheKey, pmd); loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index 7039d6325..6d1f572f9 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -769,7 +769,7 @@ private String ensureSQLSyntax(String sql) throws SQLServerException { Sha1HashKey cacheKey = new Sha1HashKey(sql); // Check for cached SQL metadata. - ParsedSQLCacheItem cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); + ParsedSQLMetadata cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); // Retrieve from cache item. procedureName = cacheItem.procedureName; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index ff162e70a..b45c698a4 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -60,10 +60,6 @@ private int executeSQLReturnFirstInt(SQLServerConnection conn, String sql) throw public void testBatchedUnprepare() throws SQLException { SQLServerConnection conOuter = null; - // Make sure correct settings are used. - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); - try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { conOuter = con; @@ -144,10 +140,6 @@ public void testBatchedUnprepare() throws SQLException { */ @Test public void testStatementPooling() throws SQLException { - // Make sure correct settings are used. - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); - try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { // Test behvaior with statement pooling. @@ -169,7 +161,7 @@ public void testStatementPooling() throws SQLException { try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { pstmt.execute(); // sp_prepexec pstmt.getMoreResults(); // Make sure handle is updated. - + handle = pstmt.getPreparedStatementHandle(); assertNotSame(0, handle); } @@ -201,9 +193,6 @@ public void testStatementPooling() throws SQLException { */ @Test public void testStatementPoolingEviction() throws SQLException { - // Make sure correct settings are used. - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); for (int testNo = 0; testNo < 2; ++testNo) { try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { @@ -263,16 +252,16 @@ public void testStatementPoolingEviction() throws SQLException { assertSame(0, con.getDiscardedServerPreparedStatementCount()); // Set statement pool size to 0 and verify statements get discarded. - int statementsInCache = con.getStatementPoolingCacheEntryCount(); + int statementsInCache = con.getStatementHandleCacheEntryCount(); con.setStatementPoolingCacheSize(0); - assertSame(0, con.getStatementPoolingCacheEntryCount()); + assertSame(0, con.getStatementHandleCacheEntryCount()); if(0 == testNo) // Verify statements moved over to discard action queue. assertSame(statementsInCache, con.getDiscardedServerPreparedStatementCount()); // Run discard actions (otherwise run on pstmt.close) - con.closeDiscardedServerPreparedStatements(); + con.closeUnreferencedPreparedStatementHandles(); assertSame(0, con.getDiscardedServerPreparedStatementCount()); @@ -281,7 +270,7 @@ public void testStatementPoolingEviction() throws SQLException { pstmt.execute(); // sp_executesql pstmt.execute(); // sp_prepexec, actual handle created and cached. - assertSame(0, con.getStatementPoolingCacheEntryCount()); + assertSame(0, con.getStatementHandleCacheEntryCount()); } } } @@ -290,9 +279,6 @@ public void testStatementPoolingEviction() throws SQLException { @Test public void testPrepareRace() throws Exception { - // Make sure correct settings are used. - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(true); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(2); String[] queries = new String[3]; queries[0] = String.format("SELECT * FROM sys.tables -- %s", UUID.randomUUID()); @@ -318,20 +304,15 @@ public void testPrepareRace() throws Exception { } threadPool.shutdown(); - threadPool.awaitTermination(12000, SECONDS); + threadPool.awaitTermination(20, SECONDS); assertNull(exception.get()); // Force un-prepares. - con.closeDiscardedServerPreparedStatements(); + con.closeUnreferencedPreparedStatementHandles(); // Verify that queue is now empty. assertSame(0, con.getDiscardedServerPreparedStatementCount()); - - } - finally { - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); } } @@ -343,26 +324,16 @@ public void testPrepareRace() throws Exception { @Test public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws SQLException { - // Verify initial defaults are correct: - assertTrue(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold() > 1); - assertTrue(false == SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - assertSame(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold(), SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); - assertSame(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall(), SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); - // Test Data Source properties SQLServerDataSource dataSource = new SQLServerDataSource(); dataSource.setURL(connectionString); // Verify defaults. assertTrue(0 < dataSource.getStatementPoolingCacheSize()); - assertSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), dataSource.getEnablePrepareOnFirstPreparedStatementCall()); - assertSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), dataSource.getServerPreparedStatementDiscardThreshold()); // Verify change dataSource.setStatementPoolingCacheSize(0); assertSame(0, dataSource.getStatementPoolingCacheSize()); dataSource.setEnablePrepareOnFirstPreparedStatementCall(!dataSource.getEnablePrepareOnFirstPreparedStatementCall()); - assertNotSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), dataSource.getEnablePrepareOnFirstPreparedStatementCall()); dataSource.setServerPreparedStatementDiscardThreshold(dataSource.getServerPreparedStatementDiscardThreshold() + 1); - assertNotSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), dataSource.getServerPreparedStatementDiscardThreshold()); // Verify connection from data source has same parameters. SQLServerConnection connDataSource = (SQLServerConnection)dataSource.getConnection(); assertSame(dataSource.getStatementPoolingCacheSize(), connDataSource.getStatementPoolingCacheSize()); @@ -370,9 +341,6 @@ public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws assertSame(dataSource.getServerPreparedStatementDiscardThreshold(), connDataSource.getServerPreparedStatementDiscardThreshold()); // Test connection string properties. - // Make sure default is not same as test. - assertNotSame(true, SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); - assertNotSame(3, SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); // Test disableStatementPooling String connectionStringDisableStatementPooling = connectionString + ";disableStatementPooling=true;"; @@ -417,32 +385,7 @@ public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws // Good! } - // Change the defaults and verify change stuck. - SQLServerConnection.setDefaultEnablePrepareOnFirstPreparedStatementCall(!SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall()); - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold() - 1); - assertNotSame(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold(), SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); - assertNotSame(SQLServerConnection.getInitialDefaultEnablePrepareOnFirstPreparedStatementCall(), SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); - - // Verify invalid (negative) changes are handled correctly. - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(-1); - assertSame(0, SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); - - // Verify instance settings. - SQLServerConnection conn1 = (SQLServerConnection)DriverManager.getConnection(connectionString); - assertSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), conn1.getServerPreparedStatementDiscardThreshold()); - assertSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), conn1.getEnablePrepareOnFirstPreparedStatementCall()); - conn1.setServerPreparedStatementDiscardThreshold(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold() + 1); - conn1.setEnablePrepareOnFirstPreparedStatementCall(!SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall()); - assertNotSame(SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold(), conn1.getServerPreparedStatementDiscardThreshold()); - assertNotSame(SQLServerConnection.getDefaultEnablePrepareOnFirstPreparedStatementCall(), conn1.getEnablePrepareOnFirstPreparedStatementCall()); - - // Verify new instance not same as changed instance. - SQLServerConnection conn2 = (SQLServerConnection)DriverManager.getConnection(connectionString); - assertNotSame(conn1.getServerPreparedStatementDiscardThreshold(), conn2.getServerPreparedStatementDiscardThreshold()); - assertNotSame(conn1.getEnablePrepareOnFirstPreparedStatementCall(), conn2.getEnablePrepareOnFirstPreparedStatementCall()); - // Verify instance setting is followed. - SQLServerConnection.setDefaultServerPreparedStatementDiscardThreshold(SQLServerConnection.getInitialDefaultServerPreparedStatementDiscardThreshold()); try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { // Turn off use of prepared statement cache. @@ -451,17 +394,19 @@ public void testStatementPoolingPreparedStatementExecAndUnprepareConfig() throws String query = "/*unprepSettingsTest*/SELECT * FROM sys.objects;"; // Verify initial default is not serial: - assertTrue(1 < SQLServerConnection.getDefaultServerPreparedStatementDiscardThreshold()); + assertTrue(1 < con.getServerPreparedStatementDiscardThreshold()); // Verify first use is batched. try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { - pstmt.execute(); + pstmt.execute(); // sp_executesql + pstmt.execute(); // sp_prepexec } + // Verify that the un-prepare action was not handled immediately. assertSame(1, con.getDiscardedServerPreparedStatementCount()); // Force un-prepares. - con.closeDiscardedServerPreparedStatements(); + con.closeUnreferencedPreparedStatementHandles(); // Verify that queue is now empty. assertSame(0, con.getDiscardedServerPreparedStatementCount()); From c989b296aee21b22696d3c0126c52c0808afcab9 Mon Sep 17 00:00:00 2001 From: tobiast Date: Tue, 16 May 2017 10:22:24 -0700 Subject: [PATCH 20/34] Removed use of lamba expression in test case. --- .../unit/statement/PreparedStatementTest.java | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index b45c698a4..fdc7fb24d 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -276,6 +276,32 @@ public void testStatementPoolingEviction() throws SQLException { } } + final class TestPrepareRace implements Runnable { + + SQLServerConnection con; + String[] queries; + AtomicReference exception; + + TestPrepareRace(SQLServerConnection con, String[] queries, AtomicReference exception) { + this.con = con; + this.queries = queries; + this.exception = exception; + } + + @Override + public void run() + { + for (int j = 0; j < 500000; j++) { + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement(queries[j % 3])) { + pstmt.execute(); + } + catch (SQLException e) { + exception.set(e); + break; + } + } + } + } @Test public void testPrepareRace() throws Exception { @@ -290,21 +316,11 @@ public void testPrepareRace() throws Exception { try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { for (int i = 0; i < 4; i++) { - threadPool.execute(() -> { - for (int j = 0; j < 500000; j++) { - try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement(queries[j % 3])) { - pstmt.execute(); - } - catch (SQLException e) { - exception.set(e); - break; - } - } - }); + threadPool.execute(new TestPrepareRace(con, queries, exception)); } threadPool.shutdown(); - threadPool.awaitTermination(20, SECONDS); + threadPool.awaitTermination(10, SECONDS); assertNull(exception.get()); From 224ccd0eb878b9d59bb15c5f94e180aee0a61d8d Mon Sep 17 00:00:00 2001 From: tobiast Date: Tue, 16 May 2017 10:45:49 -0700 Subject: [PATCH 21/34] Re-run tests, random failure... --- .../sqlserver/jdbc/unit/statement/PreparedStatementTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index fdc7fb24d..ee1938cfd 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -33,7 +33,7 @@ import com.microsoft.sqlserver.testframework.AbstractTest; @RunWith(JUnitPlatform.class) -public class PreparedStatementTest extends AbstractTest { +public class PreparedStatementTest extends AbstractTest { private void executeSQL(SQLServerConnection conn, String sql) throws SQLException { Statement stmt = conn.createStatement(); stmt.execute(sql); From ed1c9fea926d96bc8022e928fcdf84a1936b8d53 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Thu, 25 May 2017 16:01:16 -0700 Subject: [PATCH 22/34] Fixed several bugs related to invalid handle re-use. --- .../sqlserver/jdbc/SQLServerConnection.java | 64 +++- .../jdbc/SQLServerPreparedStatement.java | 153 ++++++--- .../jdbc/unit/statement/RegressionTest.java | 108 ++++++ .../RegressionTestAlwaysEncrypted.java | 313 ++++++++++++++++++ 4 files changed, 573 insertions(+), 65 deletions(-) create mode 100644 src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTestAlwaysEncrypted.java diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index 427610d84..c382a7d94 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -120,8 +120,12 @@ public class SQLServerConnection implements ISQLServerConnection { static class Sha1HashKey { private byte[] bytes; + Sha1HashKey(String sql, String parametersDefinition) { + this(String.format("%s%s", sql, parametersDefinition)); + } + Sha1HashKey(String s) { - bytes = getSha1Digest().digest(s.getBytes()); + bytes = getSha1Digest().digest(s.getBytes()); } public boolean equals(Object obj) { @@ -149,14 +153,26 @@ private java.security.MessageDigest getSha1Digest() { /** * Used to keep track of an individual prepared statement handle. */ - static class PreparedStatementHandle { + class PreparedStatementHandle { private int handle = 0; private final AtomicInteger handleRefCount = new AtomicInteger(); private boolean isDirectSql; private volatile boolean hasExecutedAtLeastOnce; private volatile boolean evictedFromCache; + private volatile boolean explicitlyDiscarded; + private Sha1HashKey key; + + PreparedStatementHandle(Sha1HashKey key) { + this.key = key; + } - PreparedStatementHandle() { + PreparedStatementHandle(Sha1HashKey key, int handle, boolean isDirectSql, boolean isEvictedFromCache) { + this(key); + + this.handle = handle; + this.isDirectSql = isDirectSql; + this.setHasExecutedAtLeastOnce(true); + this.setIsEvictedFromCache(isEvictedFromCache); } /** Has the statement been evicted from the statement handle cache. */ @@ -169,6 +185,18 @@ private void setIsEvictedFromCache(boolean isEvictedFromCache) { this.evictedFromCache = isEvictedFromCache; } + /** Specify that this statement has been explicitly discarded from being used by the cache. */ + void setIsExplicitlyDiscarded() { + this.explicitlyDiscarded = true; + + evictCachedPreparedStatementHandle(this); + } + + /** Has the statement been explicitly discarded. */ + private boolean isExplicitlyDiscarded() { + return explicitlyDiscarded; + } + /** Has the statement that this instance is related to ever been executed (with or without handle) */ boolean hasExecutedAtLeastOnce() { return hasExecutedAtLeastOnce; @@ -179,18 +207,16 @@ void setHasExecutedAtLeastOnce(boolean hasExecutedAtLeastOnce) { this.hasExecutedAtLeastOnce = hasExecutedAtLeastOnce; } - PreparedStatementHandle(int handle, boolean isDirectSql, boolean isEvictedFromCache) { - this.handle = handle; - this.isDirectSql = isDirectSql; - this.setHasExecutedAtLeastOnce(true); - this.setIsEvictedFromCache(isEvictedFromCache); - } - /** Get the actual handle. */ int getHandle() { return handle; } + /** Get the cache key. */ + Sha1HashKey getKey() { + return key; + } + /** Specify the handle. * * @return @@ -230,7 +256,7 @@ private boolean isDiscarded() { } /** Returns whether this statement has an actual server handle associated with it. */ - private boolean hasHandle() { + boolean hasHandle() { return 0 < getHandle(); } @@ -241,7 +267,7 @@ private boolean hasHandle() { * true: Reference was successfully added. */ boolean tryAddReference() { - if (!hasHandle() || isDiscarded()) + if (!hasHandle() || isDiscarded() || isExplicitlyDiscarded()) return false; else { int refCount = handleRefCount.incrementAndGet(); @@ -250,7 +276,7 @@ boolean tryAddReference() { } /** Remove a reference from this handle*/ - private void removeReference() { + void removeReference() { handleRefCount.decrementAndGet(); } } @@ -277,7 +303,7 @@ static ParsedSQLMetadata getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, Str String procName = translator.getProcedureName(); // may return null boolean returnValueSyntax = translator.hasReturnValueSyntax(); int paramCount = countParams(parsedSql); - + cacheItem = new ParsedSQLMetadata(parsedSql, paramCount, procName, returnValueSyntax); parsedSQLCache.putIfAbsent(key, cacheItem); } @@ -5688,7 +5714,7 @@ final PreparedStatementHandle getOrRegisterCachedPreparedStatementHandle(Sha1Has PreparedStatementHandle cacheItem = preparedStatementHandleCache.get(key); if (null == cacheItem) { - cacheItem = new PreparedStatementHandle(); + cacheItem = new PreparedStatementHandle(key); preparedStatementHandleCache.putIfAbsent(key, cacheItem); } @@ -5705,6 +5731,14 @@ final void returnCachedPreparedStatementHandle(PreparedStatementHandle handle) { } } + /** Force eviction of prepared statement handle cache entry. */ + final void evictCachedPreparedStatementHandle(PreparedStatementHandle handle) { + if(null == handle || null == handle.getKey()) + return; + + preparedStatementHandleCache.remove(handle.getKey()); + } + // Handle closing handles when removed from cache. final class PreparedStatementCacheEvictionListener implements EvictionListener { public void onEviction(Sha1HashKey key, PreparedStatementHandle handle) { diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 5aab030a5..60b7f18e6 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -72,7 +72,7 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS private PreparedStatementHandle cachedPreparedStatementHandle; /** Hash of user supplied SQL statement used for various cache lookups */ - private Sha1HashKey cacheKey; + private Sha1HashKey sqlTextCacheKey; /** * Array with parameter names generated in buildParamTypeDefinitions For mapping encryption information to parameters, as the second result set @@ -97,16 +97,6 @@ public class SQLServerPreparedStatement extends SQLServerStatement implements IS /** The prepared statement handle returned by the server */ private int prepStmtHandle = 0; - /** Whether the prepared statement handle is used for cursors or not (then default result set) */ - private boolean isPrepStmtHandleCursorable = false; - - private boolean isPrepStmtHandleCursorable() { - return isPrepStmtHandleCursorable; - } - - private void setIsPrepStmtHandleCursorable(boolean value) { - this.isPrepStmtHandleCursorable = value; - } private void setPreparedStatementHandle(int handle) { this.prepStmtHandle = handle; @@ -134,7 +124,6 @@ private boolean hasPreparedStatementHandle() { */ private void resetPrepStmtHandle() { prepStmtHandle = 0; - setIsPrepStmtHandleCursorable(false); } /** Flag set to true when statement execution is expected to return the prepared statement handle */ @@ -182,30 +171,16 @@ String getClassNameInternal() { stmtPoolable = true; // Create a cache key for this statement. - cacheKey = new Sha1HashKey(sql); + sqlTextCacheKey = new Sha1HashKey(sql); // Parse or fetch SQL metadata from cache. - ParsedSQLMetadata parsedSQL = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); + ParsedSQLMetadata parsedSQL = getOrCreateCachedParsedSQLMetadata(sqlTextCacheKey, sql); // Retrieve meta data from cache item. procedureName = parsedSQL.procedureName; bReturnValueSyntax = parsedSQL.bReturnValueSyntax; userSQL = parsedSQL.processedSQL; initParams(parsedSQL.parameterCount); - - // See if existing prepared statement handle can be re-used. - cachedPreparedStatementHandle = connection.getOrRegisterCachedPreparedStatementHandle(cacheKey); - // If handle was found then re-use. - if (null != cachedPreparedStatementHandle && cachedPreparedStatementHandle.hasExecutedAtLeastOnce()) { - // Because sp_executesql was already called on this SQL-text use regular prep/exec pattern. - isExecutedAtLeastOnce = true; - - // If existing handle was found and we can add reference to it, use it. - if (cachedPreparedStatementHandle.tryAddReference()) { - setIsPrepStmtHandleCursorable(false); - setPreparedStatementHandle(cachedPreparedStatementHandle.getHandle()); - } - } } /** @@ -233,7 +208,7 @@ private void closePreparedHandle() { } // If no reference to a statement pool cache item is found handle unprepare actions through batching @ connection level. else if(connection.isPreparedStatementUnprepareBatchingEnabled()) { - connection.enqueueUnprepareStatementHandle(new PreparedStatementHandle(handleToClose, executedSqlDirectly, true)); + connection.enqueueUnprepareStatementHandle(connection.new PreparedStatementHandle(null, handleToClose, executedSqlDirectly, true)); } else { // Non batched behavior (same as pre batch clean-up implementation) @@ -525,15 +500,35 @@ final void doExecutePreparedStatement(PrepStmtExecCmd command) throws SQLServerE hasNewTypeDefinitions = buildPreparedStrings(inOutParam, true); } - // Start the request and detach the response reader so that we can - // continue using it after we return. - TDSWriter tdsWriter = command.startRequest(TDS.PKT_RPC); - - doPrepExec(tdsWriter, inOutParam, hasNewTypeDefinitions, hasExistingTypeDefinitions); - - ensureExecuteResultsReader(command.startResponse(getIsResponseBufferingAdaptive())); - startResults(); - getNextResult(); + // Retry execution if existing handle could not be re-used. + int attempt = 0; + while (true) { + + ++attempt; + try { + // Re-use handle if available, requires parameter definitions which are not available until here. + if (reuseCachedHandle(hasNewTypeDefinitions, 1 < attempt)) { + hasNewTypeDefinitions = false; + } + + // Start the request and detach the response reader so that we can + // continue using it after we return. + TDSWriter tdsWriter = command.startRequest(TDS.PKT_RPC); + + doPrepExec(tdsWriter, inOutParam, hasNewTypeDefinitions, hasExistingTypeDefinitions); + + ensureExecuteResultsReader(command.startResponse(getIsResponseBufferingAdaptive())); + startResults(); + getNextResult(); + } + catch(SQLException e) { + if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) + continue; + else + throw e; + } + break; + } if (EXECUTE_QUERY == executeMethod && null == resultSet) { SQLServerException.makeFromDriverError(connection, this, SQLServerException.getErrString("R_noResultset"), null, true); @@ -543,6 +538,14 @@ else if (EXECUTE_UPDATE == executeMethod && null != resultSet) { } } + /** Should the execution be retried because the re-used cached handle could not be re-used due to server side state changes? */ + private boolean retryBasedOnFailedReuseOfCachedHandle(SQLException e, int attempt) { + // Only retry based on these error codes: + // 586: The prepared statement handle %d is not valid in this context. Please verify that current database, user default schema, and ANSI_NULLS and QUOTED_IDENTIFIER set options are not changed since the handle is prepared. + // 8179: Could not find prepared statement with handle %d. + return 1 == attempt && (586 == e.getErrorCode() || 8179 == e.getErrorCode()); + } + /** * Consume the OUT parameter for the statement object itself. * @@ -567,7 +570,7 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { setPreparedStatementHandle(param.getInt(tdsReader)); // Check if a cache reference should be updated with the newly created handle, NOT for cursorable handles. - if (null != cachedPreparedStatementHandle && !isPrepStmtHandleCursorable()) { + if (null != cachedPreparedStatementHandle && !isCursorable(executeMethod)) { // Attempt to update the handle, if the update fails remove the reference to the cache item since it references a different handle. if (!cachedPreparedStatementHandle.setHandle(prepStmtHandle, executedSqlDirectly)) cachedPreparedStatementHandle = null; // Handle could not be set, treat as not cached. @@ -890,23 +893,68 @@ private void getParameterEncryptionMetadata(Parameter[] params) throws SQLServer connection.resetCurrentCommand(); } + /** Manage re-using cached handles */ + private boolean reuseCachedHandle(boolean hasNewTypeDefinitions, boolean discardCurrentCacheItem) { + + // No re-use of caching for cursorable statements (statements that WILL use sp_cursor*) + if (isCursorable(executeMethod)) + return false; + + // If current cache item should be discarded make sure it is not used again. + if (discardCurrentCacheItem && null != cachedPreparedStatementHandle) { + + if(cachedPreparedStatementHandle.hasHandle()) + cachedPreparedStatementHandle.removeReference(); + + // Make sure the cached handle does not get re-used more. + resetPrepStmtHandle(); + cachedPreparedStatementHandle.setIsExplicitlyDiscarded(); + cachedPreparedStatementHandle = null; + + return false; + } + + // New type definitions and existing cached handle reference then deregister cached handle. + if(hasNewTypeDefinitions) { + if (null != cachedPreparedStatementHandle && hasPreparedStatementHandle() && getPreparedStatementHandle() == cachedPreparedStatementHandle.getHandle()) { + cachedPreparedStatementHandle.removeReference(); + } + cachedPreparedStatementHandle = null; + } + + // Check for new cache reference. + if (null == cachedPreparedStatementHandle) { + cachedPreparedStatementHandle = connection.getOrRegisterCachedPreparedStatementHandle(new Sha1HashKey(preparedSQL, preparedTypeDefinitions)); + + // If handle was found then re-use. + if (null != cachedPreparedStatementHandle) { + + // Because sp_executesql was already called on this SQL-text use + // regular prep/exec pattern. + if (cachedPreparedStatementHandle.hasExecutedAtLeastOnce()) + isExecutedAtLeastOnce = true; + + // If existing handle was found and we can add reference to it, use + // it. + if (cachedPreparedStatementHandle.tryAddReference()) { + setPreparedStatementHandle(cachedPreparedStatementHandle.getHandle()); + return true; + } + } + } + return false; + } + private boolean doPrepExec(TDSWriter tdsWriter, Parameter[] params, boolean hasNewTypeDefinitions, boolean hasExistingTypeDefinitions) throws SQLServerException { - // We only have a handle to re-use if it is both available and same "cursorability". - boolean isCursorableHandle = isCursorable(executeMethod); - - boolean hasHandle = hasPreparedStatementHandle() && isCursorableHandle == isPrepStmtHandleCursorable(); - + boolean hasHandle = hasPreparedStatementHandle(); boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasHandle; - // Remember cursorability. - setIsPrepStmtHandleCursorable(isCursorableHandle); - // Cursors don't use statement pooling. - if (isCursorableHandle) { + if (isCursorable(executeMethod)) { if (needsPrepare) buildServerCursorPrepExecParams(tdsWriter); @@ -2541,6 +2589,11 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th } } + // Re-use handle if available, requires parameter definitions which are not available until here. + if (reuseCachedHandle(hasNewTypeDefinitions, false)) { + hasNewTypeDefinitions = false; + } + if (numBatchesExecuted < numBatchesPrepared) { // assert null != tdsWriter; tdsWriter.writeByte((byte) nBatchStatementDelimiter); @@ -2896,7 +2949,7 @@ public final void setNull(int paramIndex, */ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws SQLServerException { - SQLServerParameterMetaData pmd = this.connection.getCachedParameterMetadata(cacheKey); + SQLServerParameterMetaData pmd = this.connection.getCachedParameterMetadata(sqlTextCacheKey); if (!forceRefresh && null != pmd) { return pmd; @@ -2906,7 +2959,7 @@ public final ParameterMetaData getParameterMetaData(boolean forceRefresh) throws checkClosed(); pmd = new SQLServerParameterMetaData(this, userSQL); - connection.registerCachedParameterMetadata(cacheKey, pmd); + connection.registerCachedParameterMetadata(sqlTextCacheKey, pmd); loggerExternal.exiting(getClassNameLogging(), "getParameterMetaData", pmd); diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java index 659141759..319dc02da 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java @@ -10,10 +10,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.sql.DriverManager; +import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Connection; +import java.sql.Statement; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; @@ -21,6 +25,7 @@ import org.junit.runner.RunWith; import com.microsoft.sqlserver.jdbc.SQLServerConnection; +import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement; import com.microsoft.sqlserver.testframework.AbstractTest; import com.microsoft.sqlserver.testframework.DBConnection; import com.microsoft.sqlserver.testframework.Utils; @@ -122,6 +127,109 @@ public void testSelectIntoUpdateCount() throws SQLException { if (null != con) con.close(); } + + /** + * Tests update query + * + * @throws SQLException + */ + @Test + public void testUpdateQuery() throws SQLException { + SQLServerConnection con = (SQLServerConnection) DriverManager.getConnection(connectionString); + String sql; + PreparedStatement pstmt = null; + JDBCType[] targets = {JDBCType.INTEGER, JDBCType.SMALLINT}; + int rows = 3; + final String tableName = "[updateQuery]"; + + Statement stmt = con.createStatement(); + Utils.dropTableIfExists(tableName, stmt); + stmt.executeUpdate("CREATE TABLE " + tableName + " (" + "c1 int null," + "PK int NOT NULL PRIMARY KEY" + ")"); + + /* + * populate table + */ + sql = "insert into " + tableName + " values(" + "?,?" + ")"; + pstmt = (connection).prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + + for (int i = 1; i <= rows; i++) { + pstmt.setObject(1, i, JDBCType.INTEGER); + pstmt.setObject(2, i, JDBCType.INTEGER); + pstmt.executeUpdate(); + } + + /* + * Update table + */ + sql = "update " + tableName + " SET c1= ? where PK =1"; + for (int i = 1; i <= rows; i++) { + pstmt = (connection).prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + for (int t = 0; t < targets.length; t++) { + pstmt.setObject(1, 5 + i, targets[t]); + pstmt.executeUpdate(); + } + } + + /* + * Verify + */ + ResultSet rs = stmt.executeQuery("select * from " + tableName); + rs.next(); + assertEquals(rs.getInt(1), 8, "Value mismatch"); + + + if (null != stmt) + stmt.close(); + if (null != con) + con.close(); + } + + private String xmlTableName = "try_SQLXML_Table"; + + /** + * Tests XML query + * + * @throws SQLException + */ + @Test + public void testXmlQuery() throws SQLException { + + Connection connection = DriverManager.getConnection(connectionString); + + Statement stmt = connection.createStatement(); + + dropTables(stmt); + createTable(stmt); + + String sql = "UPDATE " + xmlTableName + " SET [c2] = ?, [c3] = ?"; + SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(sql); + + pstmt.setObject(1, null); + pstmt.setObject(2, null, Types.SQLXML); + pstmt.executeUpdate(); + + pstmt = (SQLServerPreparedStatement) connection.prepareStatement(sql); + pstmt.setObject(1, null, Types.SQLXML); + pstmt.setObject(2, null); + pstmt.executeUpdate(); + + pstmt = (SQLServerPreparedStatement) connection.prepareStatement(sql); + pstmt.setObject(1, null); + pstmt.setObject(2, null, Types.SQLXML); + pstmt.executeUpdate(); + } + + private void dropTables(Statement stmt) throws SQLException { + stmt.executeUpdate("if object_id('" + xmlTableName + "','U') is not null" + " drop table " + xmlTableName); + } + + private void createTable(Statement stmt) throws SQLException { + + String sql = "CREATE TABLE " + xmlTableName + " ([c1] int, [c2] xml, [c3] xml)"; + + stmt.execute(sql); + } @AfterAll public static void terminate() throws SQLException { diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTestAlwaysEncrypted.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTestAlwaysEncrypted.java new file mode 100644 index 000000000..8fe6d0f9a --- /dev/null +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTestAlwaysEncrypted.java @@ -0,0 +1,313 @@ +/* + * Microsoft JDBC Driver for SQL Server + * + * Copyright(c) Microsoft Corporation All rights reserved. + * + * This program is made available under the terms of the MIT License. See the LICENSE file in the project root for more information. + */ +/* TODO: Make possible to run automated (including certs, only works on Windows now etc.)*/ +/* +package com.microsoft.sqlserver.jdbc.unit.statement; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.junit.jupiter.api.Test; +import org.junit.platform.runner.JUnitPlatform; +import org.junit.runner.RunWith; + +import com.microsoft.sqlserver.jdbc.SQLServerConnection; +import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement; +import com.microsoft.sqlserver.jdbc.SQLServerResultSet; +import com.microsoft.sqlserver.testframework.AbstractTest; + +@RunWith(JUnitPlatform.class) +public class RegressionTestAlwaysEncrypted extends AbstractTest { + String dateTable = "DateTable"; + String charTable = "CharTable"; + String numericTable = "NumericTable"; + Statement stmt = null; + Connection connection = null; + Date date; + String cekName = "CEK_Auto1"; // you need to change this to your CEK + long dateValue = 212921879801519L; + + @Test + public void alwaysEncrypted1() throws Exception { + + Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + connection = DriverManager.getConnection(connectionString + ";trustservercertificate=true;columnEncryptionSetting=enabled;database=Tobias;"); + assertTrue(null != connection); + + stmt = ((SQLServerConnection) connection).createStatement(); + + date = new Date(dateValue); + + dropTable(); + createNumericTable(); + populateNumericTable(); + printNumericTable(); + + dropTable(); + createDateTable(); + populateDateTable(); + printDateTable(); + + dropTable(); + createNumericTable(); + populateNumericTableWithNull(); + printNumericTable(); + } + + @Test + public void alwaysEncrypted2() throws Exception { + + Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); + connection = DriverManager.getConnection(connectionString + ";trustservercertificate=true;columnEncryptionSetting=enabled;database=Tobias;"); + assertTrue(null != connection); + + stmt = ((SQLServerConnection) connection).createStatement(); + + date = new Date(dateValue); + + dropTable(); + createCharTable(); + populateCharTable(); + printCharTable(); + + dropTable(); + createDateTable(); + populateDateTable(); + printDateTable(); + + dropTable(); + createNumericTable(); + populateNumericTableSpecificSetter(); + printNumericTable(); + + } + + private void populateDateTable() { + + try { + String sql = "insert into " + dateTable + " values( " + "?" + ")"; + SQLServerPreparedStatement sqlPstmt = (SQLServerPreparedStatement) ((SQLServerConnection) connection).prepareStatement(sql, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + sqlPstmt.setObject(1, date); + sqlPstmt.executeUpdate(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + private void populateCharTable() { + + try { + String sql = "insert into " + charTable + " values( " + "?,?,?,?,?,?" + ")"; + SQLServerPreparedStatement sqlPstmt = (SQLServerPreparedStatement) ((SQLServerConnection) connection).prepareStatement(sql, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + sqlPstmt.setObject(1, "hi"); + sqlPstmt.setObject(2, "sample"); + sqlPstmt.setObject(3, "hey"); + sqlPstmt.setObject(4, "test"); + sqlPstmt.setObject(5, "hello"); + sqlPstmt.setObject(6, "caching"); + sqlPstmt.executeUpdate(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + private void populateNumericTable() throws Exception { + String sql = "insert into " + numericTable + " values( " + "?,?,?,?,?,?,?,?,?" + ")"; + SQLServerPreparedStatement sqlPstmt = (SQLServerPreparedStatement) ((SQLServerConnection) connection).prepareStatement(sql, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + sqlPstmt.setObject(1, true); + sqlPstmt.setObject(2, false); + sqlPstmt.setObject(3, true); + + Integer value = 255; + sqlPstmt.setObject(4, value.shortValue(), JDBCType.TINYINT); + sqlPstmt.setObject(5, value.shortValue(), JDBCType.TINYINT); + sqlPstmt.setObject(6, value.shortValue(), JDBCType.TINYINT); + + sqlPstmt.setObject(7, Short.valueOf("1"), JDBCType.SMALLINT); + sqlPstmt.setObject(8, Short.valueOf("2"), JDBCType.SMALLINT); + sqlPstmt.setObject(9, Short.valueOf("3"), JDBCType.SMALLINT); + + sqlPstmt.executeUpdate(); + } + + private void populateNumericTableSpecificSetter() { + + try { + String sql = "insert into " + numericTable + " values( " + "?,?,?,?,?,?,?,?,?" + ")"; + SQLServerPreparedStatement sqlPstmt = (SQLServerPreparedStatement) ((SQLServerConnection) connection).prepareStatement(sql, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + sqlPstmt.setBoolean(1, true); + sqlPstmt.setBoolean(2, false); + sqlPstmt.setBoolean(3, true); + + Integer value = 255; + sqlPstmt.setShort(4, value.shortValue()); + sqlPstmt.setShort(5, value.shortValue()); + sqlPstmt.setShort(6, value.shortValue()); + + sqlPstmt.setByte(7, Byte.valueOf("127")); + sqlPstmt.setByte(8, Byte.valueOf("127")); + sqlPstmt.setByte(9, Byte.valueOf("127")); + + sqlPstmt.executeUpdate(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + private void populateNumericTableWithNull() { + + try { + String sql = "insert into " + numericTable + " values( " + "?,?,?" + ",?,?,?" + ",?,?,?" + ")"; + SQLServerPreparedStatement sqlPstmt = (SQLServerPreparedStatement) ((SQLServerConnection) connection).prepareStatement(sql, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); + sqlPstmt.setObject(1, null, java.sql.Types.BIT); + sqlPstmt.setObject(2, null, java.sql.Types.BIT); + sqlPstmt.setObject(3, null, java.sql.Types.BIT); + + sqlPstmt.setObject(4, null, java.sql.Types.TINYINT); + sqlPstmt.setObject(5, null, java.sql.Types.TINYINT); + sqlPstmt.setObject(6, null, java.sql.Types.TINYINT); + + sqlPstmt.setObject(7, null, java.sql.Types.SMALLINT); + sqlPstmt.setObject(8, null, java.sql.Types.SMALLINT); + sqlPstmt.setObject(9, null, java.sql.Types.SMALLINT); + + sqlPstmt.executeUpdate(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + private void printDateTable() throws SQLException { + + stmt = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE); + SQLServerResultSet rs = (SQLServerResultSet) stmt.executeQuery("select * from " + dateTable); + + while (rs.next()) { + System.out.println(rs.getObject(1)); + } + } + + private void printCharTable() throws SQLException { + stmt = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE); + SQLServerResultSet rs = (SQLServerResultSet) stmt.executeQuery("select * from " + charTable); + + while (rs.next()) { + System.out.println(rs.getObject(1)); + System.out.println(rs.getObject(2)); + System.out.println(rs.getObject(3)); + System.out.println(rs.getObject(4)); + System.out.println(rs.getObject(5)); + System.out.println(rs.getObject(6)); + } + + } + + private void printNumericTable() throws SQLException { + stmt = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE); + SQLServerResultSet rs = (SQLServerResultSet) stmt.executeQuery("select * from " + numericTable); + + while (rs.next()) { + System.out.println(rs.getObject(1)); + System.out.println(rs.getObject(2)); + System.out.println(rs.getObject(3)); + System.out.println(rs.getObject(4)); + System.out.println(rs.getObject(5)); + System.out.println(rs.getObject(6)); + } + + } + + private void createDateTable() throws SQLException { + + String sql = "create table " + dateTable + " (" + + "RandomizedDate date ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + ");"; + + try { + stmt.execute(sql); + } + catch (SQLException e) { + System.out.println(e); + } + } + + private void createCharTable() throws SQLException { + String sql = "create table " + charTable + " (" + "PlainChar char(20) null," + + "RandomizedChar char(20) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + "DeterministicChar char(20) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + + "PlainVarchar varchar(50) null," + + "RandomizedVarchar varchar(50) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + "DeterministicVarchar varchar(50) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + + ");"; + + try { + stmt.execute(sql); + } + catch (SQLException e) { + System.out.println(e.getMessage()); + } + } + + private void createNumericTable() throws SQLException { + String sql = "create table " + numericTable + " (" + "PlainBit bit null," + + "RandomizedBit bit ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + "DeterministicBit bit ENCRYPTED WITH (ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + + "PlainTinyint tinyint null," + + "RandomizedTinyint tinyint ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + "DeterministicTinyint tinyint ENCRYPTED WITH (ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + + "PlainSmallint smallint null," + + "RandomizedSmallint smallint ENCRYPTED WITH (ENCRYPTION_TYPE = RANDOMIZED, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + "DeterministicSmallint smallint ENCRYPTED WITH (ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = 'AEAD_AES_256_CBC_HMAC_SHA_256', COLUMN_ENCRYPTION_KEY = " + + cekName + ") NULL," + + + ");"; + + try { + stmt.execute(sql); + } + catch (SQLException e) { + System.out.println(e.getMessage()); + } + } + + private void dropTable() throws SQLException { + stmt.executeUpdate("if object_id('" + dateTable + "','U') is not null" + " drop table " + dateTable); + stmt.executeUpdate("if object_id('" + charTable + "','U') is not null" + " drop table " + charTable); + stmt.executeUpdate("if object_id('" + numericTable + "','U') is not null" + " drop table " + numericTable); + } +} +*/ \ No newline at end of file From ba3bae2af1e8515d2e7aaebdaeeafdf5a3b0880e Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Thu, 25 May 2017 16:46:02 -0700 Subject: [PATCH 23/34] Fixed test case. --- .../sqlserver/jdbc/unit/statement/RegressionTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java index 319dc02da..de6d601b4 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java @@ -137,7 +137,7 @@ public void testSelectIntoUpdateCount() throws SQLException { public void testUpdateQuery() throws SQLException { SQLServerConnection con = (SQLServerConnection) DriverManager.getConnection(connectionString); String sql; - PreparedStatement pstmt = null; + SQLServerPreparedStatement pstmt = null; JDBCType[] targets = {JDBCType.INTEGER, JDBCType.SMALLINT}; int rows = 3; final String tableName = "[updateQuery]"; @@ -150,7 +150,7 @@ public void testUpdateQuery() throws SQLException { * populate table */ sql = "insert into " + tableName + " values(" + "?,?" + ")"; - pstmt = (connection).prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, + pstmt = (SQLServerPreparedStatement)con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, connection.getHoldability()); for (int i = 1; i <= rows; i++) { @@ -164,7 +164,7 @@ public void testUpdateQuery() throws SQLException { */ sql = "update " + tableName + " SET c1= ? where PK =1"; for (int i = 1; i <= rows; i++) { - pstmt = (connection).prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + pstmt = (SQLServerPreparedStatement)con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); for (int t = 0; t < targets.length; t++) { pstmt.setObject(1, 5 + i, targets[t]); pstmt.executeUpdate(); From 970652e4f479f90f816194a2336c45dd2ba867c6 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Thu, 25 May 2017 17:05:50 -0700 Subject: [PATCH 24/34] Updated test case to run for JDBC42 --- .../sqlserver/jdbc/unit/statement/RegressionTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java index de6d601b4..7748e998b 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/RegressionTest.java @@ -8,6 +8,7 @@ package com.microsoft.sqlserver.jdbc.unit.statement; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.sql.DriverManager; import java.sql.JDBCType; @@ -135,6 +136,8 @@ public void testSelectIntoUpdateCount() throws SQLException { */ @Test public void testUpdateQuery() throws SQLException { + assumeTrue("JDBC41".equals(Utils.getConfiguredProperty("JDBC_Version")), "Aborting test case as JDBC version is not compatible. "); + SQLServerConnection con = (SQLServerConnection) DriverManager.getConnection(connectionString); String sql; SQLServerPreparedStatement pstmt = null; @@ -194,6 +197,7 @@ public void testUpdateQuery() throws SQLException { */ @Test public void testXmlQuery() throws SQLException { + assumeTrue("JDBC41".equals(Utils.getConfiguredProperty("JDBC_Version")), "Aborting test case as JDBC version is not compatible. "); Connection connection = DriverManager.getConnection(connectionString); From 20498ebc90d571f3a11e19e3d98ce045c9c141cf Mon Sep 17 00:00:00 2001 From: tobiast Date: Fri, 26 May 2017 10:55:13 -0700 Subject: [PATCH 25/34] Moved com.googlecode to mssql.googlecode to avoid naming conflicts --- .../com/microsoft/sqlserver/jdbc/SQLServerConnection.java | 6 +++--- .../concurrentlinkedhashmap/ConcurrentLinkedHashMap.java | 8 ++++---- .../googlecode/concurrentlinkedhashmap/EntryWeigher.java | 2 +- .../concurrentlinkedhashmap/EvictionListener.java | 2 +- .../googlecode/concurrentlinkedhashmap/LICENSE | 0 .../googlecode/concurrentlinkedhashmap/LinkedDeque.java | 2 +- .../googlecode/concurrentlinkedhashmap/NOTICE | 0 .../googlecode/concurrentlinkedhashmap/Weigher.java | 2 +- .../googlecode/concurrentlinkedhashmap/Weighers.java | 4 ++-- .../googlecode/concurrentlinkedhashmap/package-info.java | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java (99%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/EntryWeigher.java (96%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/EvictionListener.java (97%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/LICENSE (100%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/LinkedDeque.java (99%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/NOTICE (100%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/Weigher.java (96%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/Weighers.java (98%) rename src/main/java/{com => mssql}/googlecode/concurrentlinkedhashmap/package-info.java (97%) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index c382a7d94..e871db731 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -56,9 +56,9 @@ import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder; -import com.googlecode.concurrentlinkedhashmap.EvictionListener; +import mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder; +import mssql.googlecode.concurrentlinkedhashmap.EvictionListener; /** * SQLServerConnection implements a JDBC connection to SQL Server. SQLServerConnections support JDBC connection pooling and may be either physical diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java similarity index 99% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java index 93079f93e..36d5cc752 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; -import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.IDLE; -import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.PROCESSING; -import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.REQUIRED; +import static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.IDLE; +import static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.PROCESSING; +import static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.REQUIRED; import static java.util.Collections.emptyList; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/EntryWeigher.java similarity index 96% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/EntryWeigher.java index d07423c2e..9bf2a22b0 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/EntryWeigher.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/EntryWeigher.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; /** * A class that can determine the weight of an entry. The total weight threshold diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/EvictionListener.java similarity index 97% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/EvictionListener.java index 6b3ac196d..65488587c 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/EvictionListener.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/EvictionListener.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; /** * A listener registered for notification when an entry is evicted. An instance diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/LICENSE similarity index 100% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/LICENSE rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/LICENSE diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/LinkedDeque.java similarity index 99% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/LinkedDeque.java index 0354a69f6..2bb23ea78 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/LinkedDeque.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/LinkedDeque.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; import java.util.AbstractCollection; import java.util.Collection; diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/NOTICE similarity index 100% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/NOTICE rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/NOTICE diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weigher.java similarity index 96% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weigher.java index 2fef7f0e7..529622c8e 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weigher.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weigher.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; /** * A class that can determine the weight of a value. The total weight threshold diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weighers.java similarity index 98% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weighers.java index c3c11a152..2c5d52eb4 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/Weighers.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/Weighers.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; -import static com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.checkNotNull; +import static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.checkNotNull; import java.io.Serializable; import java.util.Collection; diff --git a/src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/package-info.java similarity index 97% rename from src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java rename to src/main/java/mssql/googlecode/concurrentlinkedhashmap/package-info.java index 57bab113b..c492a8bd3 100644 --- a/src/main/java/com/googlecode/concurrentlinkedhashmap/package-info.java +++ b/src/main/java/mssql/googlecode/concurrentlinkedhashmap/package-info.java @@ -38,4 +38,4 @@ * @see * http://code.google.com/p/concurrentlinkedhashmap/ */ -package com.googlecode.concurrentlinkedhashmap; +package mssql.googlecode.concurrentlinkedhashmap; From e4bc0d8f646787b415c607ab80e60c2c5d176c11 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Fri, 26 May 2017 15:00:16 -0700 Subject: [PATCH 26/34] getPreparedStatementHandle() should not be allowed with closed statement --- .../sqlserver/jdbc/SQLServerPreparedStatement.java | 9 +++++---- .../jdbc/unit/statement/PreparedStatementTest.java | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 60b7f18e6..dfd379574 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -107,7 +107,8 @@ private void setPreparedStatementHandle(int handle) { * @return * Per the description. */ - public int getPreparedStatementHandle() { + public int getPreparedStatementHandle() throws SQLServerException { + checkClosed(); return prepStmtHandle; } @@ -195,11 +196,11 @@ private void closePreparedHandle() { // on the server anyway. if (connection.isSessionUnAvailable()) { if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) - loggerExternal.finer(this + ": Not closing PreparedHandle:" + getPreparedStatementHandle() + "; connection is already closed."); + loggerExternal.finer(this + ": Not closing PreparedHandle:" + prepStmtHandle + "; connection is already closed."); } else { isExecutedAtLeastOnce = false; - final int handleToClose = getPreparedStatementHandle(); + final int handleToClose = prepStmtHandle; resetPrepStmtHandle(); // Handle unprepare actions through statement pooling. @@ -916,7 +917,7 @@ private boolean reuseCachedHandle(boolean hasNewTypeDefinitions, boolean discard // New type definitions and existing cached handle reference then deregister cached handle. if(hasNewTypeDefinitions) { - if (null != cachedPreparedStatementHandle && hasPreparedStatementHandle() && getPreparedStatementHandle() == cachedPreparedStatementHandle.getHandle()) { + if (null != cachedPreparedStatementHandle && hasPreparedStatementHandle() && prepStmtHandle == cachedPreparedStatementHandle.getHandle()) { cachedPreparedStatementHandle.removeReference(); } cachedPreparedStatementHandle = null; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index ee1938cfd..df6756570 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -176,13 +176,22 @@ public void testStatementPooling() throws SQLException { } // Execute new statement with different SQL text and verify it does NOT get same handle (should now fall back to using sp_executesql). + SQLServerPreparedStatement outer = null; try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query + ";")) { + outer = pstmt; pstmt.execute(); // sp_executesql pstmt.getMoreResults(); // Make sure handle is updated. assertSame(0, pstmt.getPreparedStatementHandle()); assertNotSame(handle, pstmt.getPreparedStatementHandle()); } + try { + System.out.println(outer.getPreparedStatementHandle()); + fail("Error for invalid use of getPreparedStatementHandle() after statement close expected."); + } + catch(Exception e) { + // Good! + } } } From 3f44946920beb033b249e5e60b8e85c987e951e1 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Mon, 29 May 2017 16:31:23 -0700 Subject: [PATCH 27/34] Fix for missing handle in batching case. --- .../jdbc/SQLServerPreparedStatement.java | 152 ++++++++++-------- 1 file changed, 86 insertions(+), 66 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index dfd379574..bff020128 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2590,78 +2590,98 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th } } - // Re-use handle if available, requires parameter definitions which are not available until here. - if (reuseCachedHandle(hasNewTypeDefinitions, false)) { - hasNewTypeDefinitions = false; - } - - if (numBatchesExecuted < numBatchesPrepared) { - // assert null != tdsWriter; - tdsWriter.writeByte((byte) nBatchStatementDelimiter); - } - else { - resetForReexecute(); - tdsWriter = batchCommand.startRequest(TDS.PKT_RPC); - } + // Retry execution if existing handle could not be re-used. + int attempt = 0; + while (true) { + + ++attempt; + try { - // If we have to (re)prepare the statement then we must execute it so - // that we get back a (new) prepared statement handle to use to - // execute additional batches. - // - // We must always prepare the statement the first time through. - // But we may also need to reprepare the statement if, for example, - // the size of a batch's string parameter values changes such - // that repreparation is necessary. - ++numBatchesPrepared; - if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions, hasExistingTypeDefinitions) || numBatchesPrepared == numBatches) { - ensureExecuteResultsReader(batchCommand.startResponse(getIsResponseBufferingAdaptive())); - - while (numBatchesExecuted < numBatchesPrepared) { - // NOTE: - // When making changes to anything below, consider whether similar changes need - // to be made to Statement batch execution. - - startResults(); - - try { - // Get the first result from the batch. If there is no result for this batch - // then bail, leaving EXECUTE_FAILED in the current and remaining slots of - // the update count array. - if (!getNextResult()) - return; - - // If the result is a ResultSet (rather than an update count) then throw an - // exception for this result. The exception gets caught immediately below and - // translated into (or added to) a BatchUpdateException. - if (null != resultSet) { - SQLServerException.makeFromDriverError(connection, this, SQLServerException.getErrString("R_resultsetGeneratedForUpdate"), - null, false); - } + // Re-use handle if available, requires parameter definitions which are not available until here. + if (reuseCachedHandle(hasNewTypeDefinitions, false)) { + hasNewTypeDefinitions = false; } - catch (SQLServerException e) { - // If the failure was severe enough to close the connection or roll back a - // manual transaction, then propagate the error up as a SQLServerException - // now, rather than continue with the batch. - if (connection.isSessionUnAvailable() || connection.rolledBackTransaction()) - throw e; - - // Otherwise, the connection is OK and the transaction is still intact, - // so just record the failure for the particular batch item. - updateCount = Statement.EXECUTE_FAILED; - if (null == batchCommand.batchException) - batchCommand.batchException = e; + + if (numBatchesExecuted < numBatchesPrepared) { + // assert null != tdsWriter; + tdsWriter.writeByte((byte) nBatchStatementDelimiter); + } + else { + resetForReexecute(); + tdsWriter = batchCommand.startRequest(TDS.PKT_RPC); } - // In batch execution, we have a special update count - // to indicate that no information was returned - batchCommand.updateCounts[numBatchesExecuted++] = (-1 == updateCount) ? Statement.SUCCESS_NO_INFO : updateCount; + // If we have to (re)prepare the statement then we must execute it so + // that we get back a (new) prepared statement handle to use to + // execute additional batches. + // + // We must always prepare the statement the first time through. + // But we may also need to reprepare the statement if, for example, + // the size of a batch's string parameter values changes such + // that repreparation is necessary. + if(1 == attempt) + ++numBatchesPrepared; + + if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions, hasExistingTypeDefinitions) || numBatchesPrepared == numBatches) { + ensureExecuteResultsReader(batchCommand.startResponse(getIsResponseBufferingAdaptive())); + + while (numBatchesExecuted < numBatchesPrepared) { + // NOTE: + // When making changes to anything below, consider whether similar changes need + // to be made to Statement batch execution. + + startResults(); + + try { + // Get the first result from the batch. If there is no result for this batch + // then bail, leaving EXECUTE_FAILED in the current and remaining slots of + // the update count array. + if (!getNextResult()) + return; + + // If the result is a ResultSet (rather than an update count) then throw an + // exception for this result. The exception gets caught immediately below and + // translated into (or added to) a BatchUpdateException. + if (null != resultSet) { + SQLServerException.makeFromDriverError(connection, this, SQLServerException.getErrString("R_resultsetGeneratedForUpdate"), + null, false); + } + } + catch (SQLServerException e) { + // If the failure was severe enough to close the connection or roll back a + // manual transaction, then propagate the error up as a SQLServerException + // now, rather than continue with the batch. + if (connection.isSessionUnAvailable() || connection.rolledBackTransaction()) + throw e; + + // Otherwise, the connection is OK and the transaction is still intact, + // so just record the failure for the particular batch item. + updateCount = Statement.EXECUTE_FAILED; + if (null == batchCommand.batchException) + batchCommand.batchException = e; + } + + // In batch execution, we have a special update count + // to indicate that no information was returned + batchCommand.updateCounts[numBatchesExecuted] = (-1 == updateCount) ? Statement.SUCCESS_NO_INFO : updateCount; + if(1 == attempt) + numBatchesExecuted++; + + processBatch(); + } - processBatch(); + // Only way to proceed with preparing the next set of batches is if + // we successfully executed the previously prepared set. + assert numBatchesExecuted == numBatchesPrepared; + } } - - // Only way to proceed with preparing the next set of batches is if - // we successfully executed the previously prepared set. - assert numBatchesExecuted == numBatchesPrepared; + catch(SQLException e) { + if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) + continue; + else + throw e; + } + break; } } } From ec96059d4d3a6156791f83291c765a739bf983a5 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Tue, 30 May 2017 15:42:50 -0700 Subject: [PATCH 28/34] Attempt 2 to fix batching issue with statement caching. --- .../sqlserver/jdbc/SQLServerPreparedStatement.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index bff020128..d16af0cff 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2625,6 +2625,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions, hasExistingTypeDefinitions) || numBatchesPrepared == numBatches) { ensureExecuteResultsReader(batchCommand.startResponse(getIsResponseBufferingAdaptive())); + boolean retry = false; while (numBatchesExecuted < numBatchesPrepared) { // NOTE: // When making changes to anything below, consider whether similar changes need @@ -2654,6 +2655,12 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th if (connection.isSessionUnAvailable() || connection.rolledBackTransaction()) throw e; + // Retry if invalid handle exception. + if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) { + retry = true; + break; + } + // Otherwise, the connection is OK and the transaction is still intact, // so just record the failure for the particular batch item. updateCount = Statement.EXECUTE_FAILED; @@ -2669,6 +2676,8 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th processBatch(); } + if(retry) + continue; // Only way to proceed with preparing the next set of batches is if // we successfully executed the previously prepared set. From bac85ffa443e6ee3a58584f74b1691eb327f0c90 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Tue, 30 May 2017 15:57:10 -0700 Subject: [PATCH 29/34] Testing only --- .../microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index d16af0cff..a5e222b49 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2597,11 +2597,13 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th ++attempt; try { + /* // Re-use handle if available, requires parameter definitions which are not available until here. if (reuseCachedHandle(hasNewTypeDefinitions, false)) { hasNewTypeDefinitions = false; } - + */ + if (numBatchesExecuted < numBatchesPrepared) { // assert null != tdsWriter; tdsWriter.writeByte((byte) nBatchStatementDelimiter); From 38e187ff2302fbbcd541aa7cb88fcc0eb0556fa4 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Tue, 30 May 2017 16:02:30 -0700 Subject: [PATCH 30/34] Attempt 3 (and a half)... :-/ (final Dragon) --- .../sqlserver/jdbc/SQLServerPreparedStatement.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index a5e222b49..bed669f83 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2597,12 +2597,10 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th ++attempt; try { - /* // Re-use handle if available, requires parameter definitions which are not available until here. if (reuseCachedHandle(hasNewTypeDefinitions, false)) { hasNewTypeDefinitions = false; } - */ if (numBatchesExecuted < numBatchesPrepared) { // assert null != tdsWriter; @@ -2673,10 +2671,10 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th // In batch execution, we have a special update count // to indicate that no information was returned batchCommand.updateCounts[numBatchesExecuted] = (-1 == updateCount) ? Statement.SUCCESS_NO_INFO : updateCount; + processBatch(); + if(1 == attempt) numBatchesExecuted++; - - processBatch(); } if(retry) continue; From abc1a641fe4764b511075c29e522b0e4d55d5233 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Tue, 30 May 2017 16:11:29 -0700 Subject: [PATCH 31/34] Attempt 4... --- .../microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index bed669f83..9451531a0 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2602,7 +2602,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th hasNewTypeDefinitions = false; } - if (numBatchesExecuted < numBatchesPrepared) { + if (numBatchesExecuted < numBatchesPrepared && 1 == attempt) { // assert null != tdsWriter; tdsWriter.writeByte((byte) nBatchStatementDelimiter); } From 95f5a27aaacac122dbe5280c53ecd2051b55c893 Mon Sep 17 00:00:00 2001 From: Tobias Ternstrom Date: Tue, 30 May 2017 16:24:21 -0700 Subject: [PATCH 32/34] Actual fix :-) --- .../microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 9451531a0..829a86a50 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -2598,7 +2598,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th try { // Re-use handle if available, requires parameter definitions which are not available until here. - if (reuseCachedHandle(hasNewTypeDefinitions, false)) { + if (reuseCachedHandle(hasNewTypeDefinitions, 1 < attempt)) { hasNewTypeDefinitions = false; } From 4bf4c0c1449d476ce7645a5993f5e2cfe8d648f8 Mon Sep 17 00:00:00 2001 From: tobiast Date: Wed, 31 May 2017 10:56:04 -0700 Subject: [PATCH 33/34] Finally fixed batching error with prepared handle including repro test --- .../jdbc/SQLServerPreparedStatement.java | 31 +++++++++---------- .../unit/statement/PreparedStatementTest.java | 29 +++++++++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index 829a86a50..b0755b06a 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -502,11 +502,8 @@ final void doExecutePreparedStatement(PrepStmtExecCmd command) throws SQLServerE } // Retry execution if existing handle could not be re-used. - int attempt = 0; - while (true) { - - ++attempt; - try { + for(int attempt = 1; attempt <= 2; ++attempt) { + try { // Re-use handle if available, requires parameter definitions which are not available until here. if (reuseCachedHandle(hasNewTypeDefinitions, 1 < attempt)) { hasNewTypeDefinitions = false; @@ -544,7 +541,8 @@ private boolean retryBasedOnFailedReuseOfCachedHandle(SQLException e, int attemp // Only retry based on these error codes: // 586: The prepared statement handle %d is not valid in this context. Please verify that current database, user default schema, and ANSI_NULLS and QUOTED_IDENTIFIER set options are not changed since the handle is prepared. // 8179: Could not find prepared statement with handle %d. - return 1 == attempt && (586 == e.getErrorCode() || 8179 == e.getErrorCode()); + // 99586: Error used for testing. + return 1 == attempt && (586 == e.getErrorCode() || 8179 == e.getErrorCode() || 99586 == e.getErrorCode()); } /** @@ -2591,10 +2589,8 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th } // Retry execution if existing handle could not be re-used. - int attempt = 0; - while (true) { - - ++attempt; + for(int attempt = 1; attempt <= 2; ++attempt) { + try { // Re-use handle if available, requires parameter definitions which are not available until here. @@ -2602,7 +2598,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th hasNewTypeDefinitions = false; } - if (numBatchesExecuted < numBatchesPrepared && 1 == attempt) { + if (numBatchesExecuted < numBatchesPrepared) { // assert null != tdsWriter; tdsWriter.writeByte((byte) nBatchStatementDelimiter); } @@ -2619,8 +2615,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th // But we may also need to reprepare the statement if, for example, // the size of a batch's string parameter values changes such // that repreparation is necessary. - if(1 == attempt) - ++numBatchesPrepared; + ++numBatchesPrepared; if (doPrepExec(tdsWriter, batchParam, hasNewTypeDefinitions, hasExistingTypeDefinitions) || numBatchesPrepared == numBatches) { ensureExecuteResultsReader(batchCommand.startResponse(getIsResponseBufferingAdaptive())); @@ -2657,6 +2652,8 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th // Retry if invalid handle exception. if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) { + // Reset number of batches prepared. + numBatchesPrepared = numBatchesExecuted; retry = true; break; } @@ -2673,8 +2670,7 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th batchCommand.updateCounts[numBatchesExecuted] = (-1 == updateCount) ? Statement.SUCCESS_NO_INFO : updateCount; processBatch(); - if(1 == attempt) - numBatchesExecuted++; + numBatchesExecuted++; } if(retry) continue; @@ -2685,8 +2681,11 @@ final void doExecutePreparedStatementBatch(PrepStmtBatchExecCmd batchCommand) th } } catch(SQLException e) { - if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) + if (retryBasedOnFailedReuseOfCachedHandle(e, attempt)) { + // Reset number of batches prepared. + numBatchesPrepared = numBatchesExecuted; continue; + } else throw e; } diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index df6756570..6cb0eb812 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -145,6 +145,35 @@ public void testStatementPooling() throws SQLException { // Test behvaior with statement pooling. con.setStatementPoolingCacheSize(10); + // Test with missing handle failures (fake). + this.executeSQL(con, "CREATE TABLE #update1 (col INT);INSERT #update1 VALUES (1);"); + this.executeSQL(con, "CREATE PROC #updateProc1 AS UPDATE #update1 SET col += 1; IF EXISTS (SELECT * FROM #update1 WHERE col % 5 = 0) THROW 99586, 'Prepared handle GAH!', 1;"); + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement("#updateProc1")) { + for (int i = 0; i < 100; ++i) { + assertSame(1, pstmt.executeUpdate()); + } + } + + // Test batching with missing handle failures (fake). + this.executeSQL(con, "CREATE TABLE #update2 (col INT);INSERT #update2 VALUES (1);"); + this.executeSQL(con, "CREATE PROC #updateProc2 AS UPDATE #update2 SET col += 1; IF EXISTS (SELECT * FROM #update2 WHERE col % 5 = 0) THROW 99586, 'Prepared handle GAH!', 1;"); + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement("#updateProc2")) { + for (int i = 0; i < 100; ++i) + pstmt.addBatch(); + + int[] updateCounts = pstmt.executeBatch(); + + // Verify update counts are correct + for (int i : updateCounts) { + assertSame(1, i); + } + } + } + + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + // Test behvaior with statement pooling. + con.setStatementPoolingCacheSize(10); + String lookupUniqueifier = UUID.randomUUID().toString(); String query = String.format("/*statementpoolingtest_%s*/SELECT * FROM sys.tables;", lookupUniqueifier); From 38da7940f7de2e8ea549b57d6790d40a195465e8 Mon Sep 17 00:00:00 2001 From: tobiast Date: Wed, 31 May 2017 15:38:59 -0700 Subject: [PATCH 34/34] Re-use "executedAtLeastOnce" across connections per Brett's suggestion --- .../sqlserver/jdbc/ParsedSQLMetadata.java | 4 +- .../sqlserver/jdbc/SQLServerConnection.java | 119 ++++++------------ .../jdbc/SQLServerPreparedStatement.java | 51 ++++---- .../sqlserver/jdbc/SQLServerStatement.java | 7 +- .../unit/statement/PreparedStatementTest.java | 54 ++++++-- 5 files changed, 111 insertions(+), 124 deletions(-) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java index f047ff71a..19c34ebec 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/ParsedSQLMetadata.java @@ -11,14 +11,14 @@ /** * Used for caching of meta data from parsed SQL text. */ -final class ParsedSQLMetadata { +final class ParsedSQLCacheItem { /** The SQL text AFTER processing. */ String processedSQL; int parameterCount; String procedureName; boolean bReturnValueSyntax; - ParsedSQLMetadata(String processedSQL, int parameterCount, String procedureName, boolean bReturnValueSyntax) { + ParsedSQLCacheItem(String processedSQL, int parameterCount, String procedureName, boolean bReturnValueSyntax) { this.processedSQL = processedSQL; this.parameterCount = parameterCount; this.procedureName = procedureName; diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java index e871db731..7ab9619d5 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java @@ -157,22 +157,16 @@ class PreparedStatementHandle { private int handle = 0; private final AtomicInteger handleRefCount = new AtomicInteger(); private boolean isDirectSql; - private volatile boolean hasExecutedAtLeastOnce; private volatile boolean evictedFromCache; private volatile boolean explicitlyDiscarded; private Sha1HashKey key; - - PreparedStatementHandle(Sha1HashKey key) { - this.key = key; - } PreparedStatementHandle(Sha1HashKey key, int handle, boolean isDirectSql, boolean isEvictedFromCache) { - this(key); - + this.key = key; this.handle = handle; this.isDirectSql = isDirectSql; - this.setHasExecutedAtLeastOnce(true); this.setIsEvictedFromCache(isEvictedFromCache); + handleRefCount.set(1); } /** Has the statement been evicted from the statement handle cache. */ @@ -197,16 +191,6 @@ private boolean isExplicitlyDiscarded() { return explicitlyDiscarded; } - /** Has the statement that this instance is related to ever been executed (with or without handle) */ - boolean hasExecutedAtLeastOnce() { - return hasExecutedAtLeastOnce; - } - - /** Specify whether the statement that this instance is related to ever been executed (with or without handle) */ - void setHasExecutedAtLeastOnce(boolean hasExecutedAtLeastOnce) { - this.hasExecutedAtLeastOnce = hasExecutedAtLeastOnce; - } - /** Get the actual handle. */ int getHandle() { return handle; @@ -217,22 +201,6 @@ Sha1HashKey getKey() { return key; } - /** Specify the handle. - * - * @return - * false: Handle could not be referenced (already references other handle). - * true: Handle was successfully set. - */ - boolean setHandle(int handle, boolean isDirectSql) { - if (handleRefCount.compareAndSet(0, 1)) { - this.handle = handle; - this.isDirectSql = isDirectSql; - return true; - } - else - return false; - } - boolean isDirectSql() { return isDirectSql; } @@ -244,10 +212,7 @@ boolean isDirectSql() { * true: Handle was successfully put on path for discarding. */ private boolean tryDiscardHandle() { - if(!hasHandle()) - return false; - else - return handleRefCount.compareAndSet(0, -999); + return handleRefCount.compareAndSet(0, -999); } /** Returns whether this statement has been discarded and can no longer be re-used. */ @@ -255,11 +220,6 @@ private boolean isDiscarded() { return 0 > handleRefCount.intValue(); } - /** Returns whether this statement has an actual server handle associated with it. */ - boolean hasHandle() { - return 0 < getHandle(); - } - /** Adds a new reference to this handle, i.e. re-using it. * * @return @@ -267,7 +227,7 @@ boolean hasHandle() { * true: Reference was successfully added. */ boolean tryAddReference() { - if (!hasHandle() || isDiscarded() || isExplicitlyDiscarded()) + if (isDiscarded() || isExplicitlyDiscarded()) return false; else { int refCount = handleRefCount.incrementAndGet(); @@ -285,31 +245,32 @@ void removeReference() { static final private int PARSED_SQL_CACHE_SIZE = 100; /** Cache of parsed SQL meta data */ - static private ConcurrentLinkedHashMap parsedSQLCache; + static private ConcurrentLinkedHashMap parsedSQLCache; static { - parsedSQLCache = new Builder() + parsedSQLCache = new Builder() .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE) .build(); } - /** Get prepared statement cache entry if exists, if not parse and create a new one */ - static ParsedSQLMetadata getOrCreateCachedParsedSQLMetadata(Sha1HashKey key, String sql) throws SQLServerException { - ParsedSQLMetadata cacheItem = parsedSQLCache.get(key); - if (null == cacheItem) { - JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); - - String parsedSql = translator.translate(sql); - String procName = translator.getProcedureName(); // may return null - boolean returnValueSyntax = translator.hasReturnValueSyntax(); - int paramCount = countParams(parsedSql); - - cacheItem = new ParsedSQLMetadata(parsedSql, paramCount, procName, returnValueSyntax); - parsedSQLCache.putIfAbsent(key, cacheItem); - } - - return cacheItem; - } + /** Get prepared statement cache entry if exists, if not parse and create a new one */ + static ParsedSQLCacheItem getCachedParsedSQL(Sha1HashKey key) { + return parsedSQLCache.get(key); + } + + /** Parse and create a information about parsed SQL text */ + static ParsedSQLCacheItem parseAndCacheSQL(Sha1HashKey key, String sql) throws SQLServerException { + JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator(); + + String parsedSql = translator.translate(sql); + String procName = translator.getProcedureName(); // may return null + boolean returnValueSyntax = translator.hasReturnValueSyntax(); + int paramCount = countParams(parsedSql); + + ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem (parsedSql, paramCount, procName, returnValueSyntax); + parsedSQLCache.putIfAbsent(key, cacheItem); + return cacheItem; + } /** Size of the prepared statement handle cache */ private int statementPoolingCacheSize = 10; @@ -5501,10 +5462,8 @@ final void enqueueUnprepareStatementHandle(PreparedStatementHandle statementHand loggerExternal.finer(this + ": Adding PreparedHandle to queue for un-prepare:" + statementHandle.getHandle()); // Add the new handle to the discarding queue and find out current # enqueued. - if(statementHandle.hasHandle()) { - this.discardedPreparedStatementHandles.add(statementHandle); - this.discardedPreparedStatementHandleCount.incrementAndGet(); - } + this.discardedPreparedStatementHandles.add(statementHandle); + this.discardedPreparedStatementHandleCount.incrementAndGet(); } @@ -5708,27 +5667,29 @@ final void registerCachedParameterMetadata(Sha1HashKey key, SQLServerParameterMe } /** Get or create prepared statement handle cache entry if statement pooling is enabled */ - final PreparedStatementHandle getOrRegisterCachedPreparedStatementHandle(Sha1HashKey key) { + final PreparedStatementHandle getCachedPreparedStatementHandle(Sha1HashKey key) { if(!isStatementPoolingEnabled()) return null; - PreparedStatementHandle cacheItem = preparedStatementHandleCache.get(key); - if (null == cacheItem) { - cacheItem = new PreparedStatementHandle(key); - preparedStatementHandleCache.putIfAbsent(key, cacheItem); - } + return preparedStatementHandleCache.get(key); + } + /** Get or create prepared statement handle cache entry if statement pooling is enabled */ + final PreparedStatementHandle registerCachedPreparedStatementHandle(Sha1HashKey key, int handle, boolean isDirectSql) { + if(!isStatementPoolingEnabled() || null == key) + return null; + + PreparedStatementHandle cacheItem = new PreparedStatementHandle(key, handle, isDirectSql, false); + preparedStatementHandleCache.putIfAbsent(key, cacheItem); return cacheItem; } /** Return prepared statement handle cache entry so it can be un-prepared. */ final void returnCachedPreparedStatementHandle(PreparedStatementHandle handle) { - if(handle.hasHandle()) { - handle.removeReference(); + handle.removeReference(); - if (handle.isEvictedFromCache() && handle.tryDiscardHandle()) - enqueueUnprepareStatementHandle(handle); - } + if (handle.isEvictedFromCache() && handle.tryDiscardHandle()) + enqueueUnprepareStatementHandle(handle); } /** Force eviction of prepared statement handle cache entry. */ @@ -5746,7 +5707,7 @@ public void onEviction(Sha1HashKey key, PreparedStatementHandle handle) { handle.setIsEvictedFromCache(true); // Mark as evicted from cache. // Only discard if not referenced. - if(handle.hasHandle() && handle.tryDiscardHandle()) { + if(handle.tryDiscardHandle()) { enqueueUnprepareStatementHandle(handle); // Do not run discard actions here! Can interfere with executing statement. } diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java index b0755b06a..881448b9b 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java @@ -8,7 +8,8 @@ package com.microsoft.sqlserver.jdbc; -import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getOrCreateCachedParsedSQLMetadata; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getCachedParsedSQL; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.parseAndCacheSQL; import java.io.InputStream; import java.io.Reader; @@ -175,7 +176,13 @@ String getClassNameInternal() { sqlTextCacheKey = new Sha1HashKey(sql); // Parse or fetch SQL metadata from cache. - ParsedSQLMetadata parsedSQL = getOrCreateCachedParsedSQLMetadata(sqlTextCacheKey, sql); + ParsedSQLCacheItem parsedSQL = getCachedParsedSQL(sqlTextCacheKey); + if(null != parsedSQL) { + isExecutedAtLeastOnce = true; + } + else { + parsedSQL = parseAndCacheSQL(sqlTextCacheKey, sql); + } // Retrieve meta data from cache item. procedureName = parsedSQL.procedureName; @@ -568,11 +575,9 @@ boolean onRetValue(TDSReader tdsReader) throws SQLServerException { setPreparedStatementHandle(param.getInt(tdsReader)); - // Check if a cache reference should be updated with the newly created handle, NOT for cursorable handles. - if (null != cachedPreparedStatementHandle && !isCursorable(executeMethod)) { - // Attempt to update the handle, if the update fails remove the reference to the cache item since it references a different handle. - if (!cachedPreparedStatementHandle.setHandle(prepStmtHandle, executedSqlDirectly)) - cachedPreparedStatementHandle = null; // Handle could not be set, treat as not cached. + // Cache the reference to the newly created handle, NOT for cursorable handles. + if (null == cachedPreparedStatementHandle && !isCursorable(executeMethod)) { + cachedPreparedStatementHandle = connection.registerCachedPreparedStatementHandle(new Sha1HashKey(preparedSQL, preparedTypeDefinitions), prepStmtHandle, executedSqlDirectly); } param.skipValue(tdsReader, true); @@ -902,8 +907,7 @@ private boolean reuseCachedHandle(boolean hasNewTypeDefinitions, boolean discard // If current cache item should be discarded make sure it is not used again. if (discardCurrentCacheItem && null != cachedPreparedStatementHandle) { - if(cachedPreparedStatementHandle.hasHandle()) - cachedPreparedStatementHandle.removeReference(); + cachedPreparedStatementHandle.removeReference(); // Make sure the cached handle does not get re-used more. resetPrepStmtHandle(); @@ -923,21 +927,16 @@ private boolean reuseCachedHandle(boolean hasNewTypeDefinitions, boolean discard // Check for new cache reference. if (null == cachedPreparedStatementHandle) { - cachedPreparedStatementHandle = connection.getOrRegisterCachedPreparedStatementHandle(new Sha1HashKey(preparedSQL, preparedTypeDefinitions)); + PreparedStatementHandle cachedHandle = connection.getCachedPreparedStatementHandle(new Sha1HashKey(preparedSQL, preparedTypeDefinitions)); // If handle was found then re-use. - if (null != cachedPreparedStatementHandle) { - - // Because sp_executesql was already called on this SQL-text use - // regular prep/exec pattern. - if (cachedPreparedStatementHandle.hasExecutedAtLeastOnce()) - isExecutedAtLeastOnce = true; - - // If existing handle was found and we can add reference to it, use - // it. - if (cachedPreparedStatementHandle.tryAddReference()) { - setPreparedStatementHandle(cachedPreparedStatementHandle.getHandle()); - return true; + if (null != cachedHandle) { + + // If existing handle was found and we can add reference to it, use it. + if (cachedHandle.tryAddReference()) { + setPreparedStatementHandle(cachedHandle.getHandle()); + cachedPreparedStatementHandle = cachedHandle; + return true; } } } @@ -949,8 +948,7 @@ private boolean doPrepExec(TDSWriter tdsWriter, boolean hasNewTypeDefinitions, boolean hasExistingTypeDefinitions) throws SQLServerException { - boolean hasHandle = hasPreparedStatementHandle(); - boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasHandle; + boolean needsPrepare = (hasNewTypeDefinitions && hasExistingTypeDefinitions) || !hasPreparedStatementHandle(); // Cursors don't use statement pooling. if (isCursorable(executeMethod)) { @@ -969,11 +967,6 @@ private boolean doPrepExec(TDSWriter tdsWriter, ) { buildExecSQLParams(tdsWriter); isExecutedAtLeastOnce = true; - - // Enable re-use if caching is on by moving to sp_prepexec on next call even from separate instance. - if (null != cachedPreparedStatementHandle) { - cachedPreparedStatementHandle.setHasExecutedAtLeastOnce(true); - } } // Second execution, use prepared statements since we seem to be re-using it. else if(needsPrepare) diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java index 6d1f572f9..81718b73e 100644 --- a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java +++ b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java @@ -8,7 +8,8 @@ package com.microsoft.sqlserver.jdbc; -import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getOrCreateCachedParsedSQLMetadata; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.getCachedParsedSQL; +import static com.microsoft.sqlserver.jdbc.SQLServerConnection.parseAndCacheSQL; import java.sql.BatchUpdateException; import java.sql.ResultSet; @@ -769,7 +770,9 @@ private String ensureSQLSyntax(String sql) throws SQLServerException { Sha1HashKey cacheKey = new Sha1HashKey(sql); // Check for cached SQL metadata. - ParsedSQLMetadata cacheItem = getOrCreateCachedParsedSQLMetadata(cacheKey, sql); + ParsedSQLCacheItem cacheItem = getCachedParsedSQL(cacheKey); + if (null == cacheItem) + cacheItem = parseAndCacheSQL(cacheKey, sql); // Retrieve from cache item. procedureName = cacheItem.procedureName; diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java index 6cb0eb812..c274e3611 100644 --- a/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java +++ b/src/test/java/com/microsoft/sqlserver/jdbc/unit/statement/PreparedStatementTest.java @@ -18,6 +18,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Random; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -31,6 +32,7 @@ import com.microsoft.sqlserver.jdbc.SQLServerDataSource; import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement; import com.microsoft.sqlserver.testframework.AbstractTest; +import com.microsoft.sqlserver.testframework.util.RandomUtil; @RunWith(JUnitPlatform.class) public class PreparedStatementTest extends AbstractTest { @@ -81,17 +83,6 @@ public void testBatchedUnprepare() throws SQLException { int iterations = 25; - // Verify no prepares for 1 time only uses. - for(int i = 0; i < iterations; ++i) { - try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { - pstmt.execute(); - } - assertSame(0, con.getDiscardedServerPreparedStatementCount()); - } - - // Verify total cache use. - assertSame(iterations, executeSQLReturnFirstInt(con, verifyTotalCacheUsesQuery)); - query = String.format("/*unpreparetest_%s, sp_executesql->sp_prepexec->sp_execute- batched sp_unprepare*/SELECT * FROM sys.tables;", lookupUniqueifier); int prevDiscardActionCount = 0; @@ -101,7 +92,7 @@ public void testBatchedUnprepare() throws SQLException { // Verify current queue depth is expected. assertSame(prevDiscardActionCount, con.getDiscardedServerPreparedStatementCount()); - try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(query)) { + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement)con.prepareStatement(String.format("%s--%s", query, i))) { pstmt.execute(); // sp_executesql pstmt.execute(); // sp_prepexec @@ -140,6 +131,45 @@ public void testBatchedUnprepare() throws SQLException { */ @Test public void testStatementPooling() throws SQLException { + // Test % handle re-use + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { + String query = String.format("/*statementpoolingtest_re-use_%s*/SELECT TOP(1) * FROM sys.tables;", UUID.randomUUID().toString()); + + con.setStatementPoolingCacheSize(10); + + boolean[] prepOnFirstCalls = {false, true}; + + for(boolean prepOnFirstCall : prepOnFirstCalls) { + + con.setEnablePrepareOnFirstPreparedStatementCall(prepOnFirstCall); + + int[] queryCounts = {10, 20, 30, 40}; + for(int queryCount : queryCounts) { + String[] queries = new String[queryCount]; + for(int i = 0; i < queries.length; ++i) { + queries[i] = String.format("%s--%s--%s--%s", query, i, queryCount, prepOnFirstCall); + } + + int testsWithHandleReuse = 0; + final int testCount = 500; + for(int i = 0; i < testCount; ++i) { + Random random = new Random(); + int queryNumber = random.nextInt(queries.length); + try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement(queries[queryNumber])) { + pstmt.execute(); + + // Grab handle-reuse before it would be populated if initially created. + if(0 < pstmt.getPreparedStatementHandle()) + testsWithHandleReuse++; + + pstmt.getMoreResults(); // Make sure handle is updated. + } + } + System.out.println(String.format("Prep on first call: %s Query count:%s: %s of %s (%s)", prepOnFirstCall, queryCount, testsWithHandleReuse, testCount, (double)testsWithHandleReuse/(double)testCount)); + } + } + } + try (SQLServerConnection con = (SQLServerConnection)DriverManager.getConnection(connectionString)) { // Test behvaior with statement pooling.