From 0e56b30d6f6b04ee9b1ca4d58d529215b65c1f99 Mon Sep 17 00:00:00 2001 From: Annie Liang <64233642+xinlian12@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:36:27 -0800 Subject: [PATCH] allowUsingHttp2 (#42947) * allow using http2 through system properties --------- Co-authored-by: annie-mac --- .../azure/cosmos/CosmosDiagnosticsTest.java | 15 +++- .../cosmos/implementation/ConfigsTests.java | 42 +++++++-- .../ConnectionStateListenerTest.java | 2 +- .../com/azure/cosmos/rx/LogLevelTest.java | 52 ++++++----- .../rx/MultiMasterConflictResolutionTest.java | 2 +- .../com/azure/cosmos/rx/ProxyHostTest.java | 12 ++- sdk/cosmos/azure-cosmos/CHANGELOG.md | 4 + .../azure/cosmos/implementation/Configs.java | 86 +++++++++++++++++-- .../implementation/RxDocumentClientImpl.java | 4 +- .../RntbdTransportClient.java | 2 +- .../SharedTransportClient.java | 2 +- .../StoreClientFactory.java | 2 +- .../http/Http2ConnectionConfig.java | 37 ++++++++ .../Http2ResponseHeaderCleanerHandler.java | 47 ++++++++++ .../implementation/http/HttpClient.java | 11 +++ .../implementation/http/HttpClientConfig.java | 11 +++ .../http/ReactorNettyClient.java | 30 ++++++- sdk/cosmos/live-http2-platform-matrix.json | 27 ++++++ sdk/cosmos/tests.yml | 34 ++++++++ 19 files changed, 373 insertions(+), 49 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ConnectionConfig.java create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java create mode 100644 sdk/cosmos/live-http2-platform-matrix.json diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index 8e427852b3a9c..249a402127c90 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -70,8 +70,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.mockito.Mockito; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -410,8 +412,19 @@ public void gatewayDiagnosticsOnException() throws Exception { } @Test(groups = {"fast"}, timeOut = TIMEOUT) - public void gatewayDiagnostgiticsOnNonCosmosException() { + public void gatewayDiagnosticsOnNonCosmosException() { CosmosAsyncClient testClient = null; + + // TODO[Http2]: Re-evaluate this test with http2 public API + boolean isHttp2Enabled = false; + if (!System.getProperty("COSMOS.HTTP2_ENABLED", StringUtils.EMPTY).isEmpty()) { + isHttp2Enabled = Boolean.parseBoolean(System.getProperty("COSMOS.HTTP2_ENABLED")); + } + + if (isHttp2Enabled) { + throw new SkipException("Test gatewayDiagnosticsOnNonCosmosException only with http1.1"); + } + try { GatewayConnectionConfig gatewayConnectionConfig = new GatewayConnectionConfig(); gatewayConnectionConfig.setMaxConnectionPoolSize(1); // using a small value to force pendingAcquisitionTimeout happen diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConfigsTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConfigsTests.java index f2c585c6d2d1e..f071799b08ccb 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConfigsTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConfigsTests.java @@ -106,12 +106,42 @@ public void emulatorHost() { } @Test(groups = { "emulator" }) - public void sslContextTest() { - Configs config = new Configs(); - SslContext sslContext = config.getSslContext(false); - assertThat(sslContext).isEqualTo(ReflectionUtils.getSslContext(config)); + public void http2Enabled() { + assertThat(Configs.isHttp2Enabled()).isFalse(); + + System.setProperty("COSMOS.HTTP2_ENABLED", "true"); + assertThat(Configs.isHttp2Enabled()).isTrue(); + + System.clearProperty("COSMOS.HTTP2_ENABLED"); + } + + @Test(groups = { "unit" }) + public void http2MaxConnectionPoolSize() { + assertThat(Configs.getHttp2MaxConnectionPoolSize()).isEqualTo(1000); + + System.setProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE", "10"); + assertThat(Configs.getHttp2MaxConnectionPoolSize()).isEqualTo(10); + + System.clearProperty("COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"); + } + + @Test(groups = { "unit" }) + public void http2MinConnectionPoolSize() { + assertThat(Configs.getHttp2MinConnectionPoolSize()).isEqualTo(1); + + System.setProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE", "10"); + assertThat(Configs.getHttp2MinConnectionPoolSize()).isEqualTo(10); + + System.clearProperty("COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"); + } + + @Test(groups = { "unit" }) + public void http2MaxConcurrentStreams() { + assertThat(Configs.getHttp2MaxConcurrentStreams()).isEqualTo(30); + + System.setProperty("COSMOS.HTTP2_MAX_CONCURRENT_STREAMS", "10"); + assertThat(Configs.getHttp2MaxConcurrentStreams()).isEqualTo(10); - sslContext = config.getSslContext(true); - assertThat(sslContext).isEqualTo(ReflectionUtils.getSslContextWithCertValidationDisabled(config)); + System.clearProperty("COSMOS.HTTP2_MAX_CONCURRENT_STREAMS"); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java index 0604c9ac3ca7c..fe329253fe969 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java @@ -91,7 +91,7 @@ public void connectionStateListener_OnConnectionEvent( SslContext sslContext = SslContextUtils.CreateSslContext("client.jks", false); Configs config = Mockito.mock(Configs.class); - Mockito.doReturn(sslContext).when(config).getSslContext(false); + Mockito.doReturn(sslContext).when(config).getSslContext(false, false); RntbdTransportClient client = new RntbdTransportClient( config, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/LogLevelTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/LogLevelTest.java index 9bcd045a627dc..ae34b7522d567 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/LogLevelTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/LogLevelTest.java @@ -6,6 +6,7 @@ import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosAsyncContainer; import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.http.ReactorNettyClient; @@ -35,10 +36,11 @@ public class LogLevelTest extends TestSuiteBase { public final static String COSMOS_DB_LOGGING_CATEGORY = "com.azure.cosmos"; public final static String NETWORK_LOGGING_CATEGORY = "com.azure.cosmos.netty-network"; - public final static String LOG_PATTERN_1 = "HTTP/1.1 201"; - public final static String LOG_PATTERN_2 = "| 0 1 2 3 4 5 6 7 8 9 a b c d e f |"; - public final static String LOG_PATTERN_3 = "USER_EVENT: SslHandshakeCompletionEvent(SUCCESS)"; - public final static String LOG_PATTERN_4 = "CONNECT: "; + public final static String LOG_PATTERN_HTTP_1_1 = "HTTP/1.1 201"; + public final static String LOG_PATTERN_HTTP_2 = "HTTP/2"; + public final static String LOG_PATTERN_1 = "| 0 1 2 3 4 5 6 7 8 9 a b c d e f |"; + public final static String LOG_PATTERN_2 = "USER_EVENT: SslHandshakeCompletionEvent(SUCCESS)"; + public final static String LOG_PATTERN_3 = "CONNECT: "; private static final String APPENDER_NAME = "StringWriterAppender"; private static CosmosAsyncContainer createdCollection; @@ -70,10 +72,9 @@ public void after_LogLevelTest() { * This test will try to create document with netty wire DEBUG logging and * validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithDebugLevel() throws Exception { + public void createDocumentWithDebugLevel() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(NETWORK_LOGGING_CATEGORY, Level.DEBUG, APPENDER_NAME, consoleWriter); @@ -90,17 +91,16 @@ public void createDocumentWithDebugLevel() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } /** * This test will try to create document with netty wire WARN logging and * validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithWarningLevel() throws Exception { + public void createDocumentWithWarningLevel() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(NETWORK_LOGGING_CATEGORY, Level.WARN, APPENDER_NAME, consoleWriter); @@ -116,17 +116,16 @@ public void createDocumentWithWarningLevel() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } /** * This test will try to create document with netty wire TRACE logging and * validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithTraceLevel() throws Exception { + public void createDocumentWithTraceLevel() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(NETWORK_LOGGING_CATEGORY, Level.TRACE, APPENDER_NAME, consoleWriter); @@ -143,11 +142,11 @@ public void createDocumentWithTraceLevel() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } @Test(groups = { "fast" }, timeOut = TIMEOUT, enabled = false) - public void createDocumentWithTraceLevelAtRoot() throws Exception { + public void createDocumentWithTraceLevelAtRoot() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(COSMOS_DB_LOGGING_CATEGORY, Level.TRACE, APPENDER_NAME, consoleWriter); @@ -164,11 +163,11 @@ public void createDocumentWithTraceLevelAtRoot() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } @Test(groups = { "fast" }, timeOut = TIMEOUT, enabled = false) - public void createDocumentWithDebugLevelAtRoot() throws Exception { + public void createDocumentWithDebugLevelAtRoot() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(COSMOS_DB_LOGGING_CATEGORY, Level.DEBUG, APPENDER_NAME, consoleWriter); @@ -185,17 +184,16 @@ public void createDocumentWithDebugLevelAtRoot() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } /** * This test will try to create document with netty wire ERROR logging and * validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithErrorLevel() throws Exception { + public void createDocumentWithErrorLevel() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(NETWORK_LOGGING_CATEGORY, Level.ERROR, APPENDER_NAME, consoleWriter); @@ -211,17 +209,16 @@ public void createDocumentWithErrorLevel() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); } /** * This test will try to create document with netty wire INFO logging and * validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithInfoLevel() throws Exception { + public void createDocumentWithInfoLevel() { final StringWriter consoleWriter = new StringWriter(); addAppenderAndLogger(NETWORK_LOGGING_CATEGORY, Level.INFO, APPENDER_NAME, consoleWriter); @@ -237,7 +234,16 @@ public void createDocumentWithInfoLevel() throws Exception { assertThat(consoleWriter.toString()).contains(LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_2); assertThat(consoleWriter.toString()).contains(LOG_PATTERN_3); - assertThat(consoleWriter.toString()).contains(LOG_PATTERN_4); + validateHttpProtocol(consoleWriter); + } + + private void validateHttpProtocol(StringWriter consoleWriter) { + boolean isHttp2Enabled = Configs.isHttp2Enabled(); + if (isHttp2Enabled) { + assertThat(consoleWriter.toString()).contains(LOG_PATTERN_HTTP_2); + } else { + assertThat(consoleWriter.toString()).contains(LOG_PATTERN_HTTP_1_1); + } } private InternalObjectNode getDocumentDefinition() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java index 6f71bf9f18f41..3298660af6fe1 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiMasterConflictResolutionTest.java @@ -82,7 +82,7 @@ public void conflictResolutionPolicyCRUD() { // when (e.StatusCode == HttpStatusCode.BadRequest) CosmosException dce = Utils.as(e, CosmosException.class); if (dce != null && dce.getStatusCode() == 400) { - assertThat(dce.getMessage()).contains("Invalid path '\\\\/a\\\\/b' for last writer wins conflict resolution"); + assertThat(dce.getMessage()).contains("Invalid path '/a/b' for last writer wins conflict resolution"); } else { throw e; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ProxyHostTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ProxyHostTest.java index 7036091aec1e2..9e3f19d766153 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ProxyHostTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ProxyHostTest.java @@ -9,6 +9,7 @@ import com.azure.cosmos.CosmosAsyncDatabase; import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.GatewayConnectionConfig; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.models.CosmosItemRequestOptions; @@ -96,10 +97,9 @@ public void createDocumentWithValidHttpProxy() throws Exception { /** * This test will try to create document via http proxy server with netty wire logging and validate it. * - * @throws Exception */ @Test(groups = { "fast" }, timeOut = TIMEOUT) - public void createDocumentWithValidHttpProxyWithNettyWireLogging() throws Exception { + public void createDocumentWithValidHttpProxyWithNettyWireLogging() { CosmosAsyncClient clientWithRightProxy = null; try { final StringWriter consoleWriter = new StringWriter(); @@ -125,7 +125,13 @@ public void createDocumentWithValidHttpProxyWithNettyWireLogging() throws Except assertThat(consoleWriter.toString()).contains(LogLevelTest.LOG_PATTERN_1); assertThat(consoleWriter.toString()).contains(LogLevelTest.LOG_PATTERN_2); - assertThat(consoleWriter.toString()).contains(LogLevelTest.LOG_PATTERN_3); + boolean isHttp2Enabled = Configs.isHttp2Enabled(); + if (isHttp2Enabled) { + assertThat(consoleWriter.toString()).contains(LogLevelTest.LOG_PATTERN_HTTP_2); + } else { + assertThat(consoleWriter.toString()).contains(LogLevelTest.LOG_PATTERN_HTTP_1_1); + } + } finally { safeClose(clientWithRightProxy); } diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 2641c091c7a9d..784f1ebf24fdb 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -9,6 +9,10 @@ #### Bugs Fixed #### Other Changes +* Added support to enable http2 for gateway mode with system property `COSMOS.HTTP2_ENABLED` and system variable `COSMOS_HTTP2_ENABLED`. - [PR 42947](https://github.com/Azure/azure-sdk-for-java/pull/42947) +* Added support to allow changing http2 max connection pool size with system property `COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE` and system variable `COSMOS_HTTP2_MAX_CONNECTION_POOL_SIZE`. - [PR 42947](https://github.com/Azure/azure-sdk-for-java/pull/42947) +* Added support to allow changing http2 max connection pool size with system property `COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE` and system variable `COSMOS_HTTP2_MIN_CONNECTION_POOL_SIZE`. - [PR 42947](https://github.com/Azure/azure-sdk-for-java/pull/42947) +* Added support to allow changing http2 max connection pool size with system property `COSMOS.HTTP2_MAX_CONCURRENT_STREAMS` and system variable `COSMOS_HTTP2_MAX_CONCURRENT_STREAMS`. - [PR 42947](https://github.com/Azure/azure-sdk-for-java/pull/42947) ### 4.65.0 (2024-11-19) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 6b5d5be14b69a..2ec99d1e2dc30 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -5,6 +5,8 @@ import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.azure.cosmos.implementation.circuitBreaker.PartitionLevelCircuitBreakerConfig; import com.azure.cosmos.implementation.directconnectivity.Protocol; +import io.netty.handler.ssl.ApplicationProtocolConfig; +import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; @@ -31,8 +33,6 @@ public class Configs { public static final String SPECULATION_TYPE = "COSMOS_SPECULATION_TYPE"; public static final String SPECULATION_THRESHOLD = "COSMOS_SPECULATION_THRESHOLD"; public static final String SPECULATION_THRESHOLD_STEP = "COSMOS_SPECULATION_THRESHOLD_STEP"; - private final SslContext sslContext; - private final SslContext sslContextWithCertValidationDisabled; // The names we use are consistent with the: // * Azure environment variable naming conventions documented at https://azure.github.io/azure-sdk/java_implementation.html and @@ -282,25 +282,53 @@ public class Configs { private static final String EMULATOR_HOST = "COSMOS.EMULATOR_HOST"; private static final String EMULATOR_HOST_VARIABLE = "COSMOS_EMULATOR_HOST"; - public Configs() { - this.sslContext = sslContextInit(false); - this.sslContextWithCertValidationDisabled = sslContextInit(true); - } + // Flag to indicate whether enabled http2 for gateway + private static final boolean DEFAULT_HTTP2_ENABLED = false; + private static final String HTTP2_ENABLED = "COSMOS.HTTP2_ENABLED"; + private static final String HTTP2_ENABLED_VARIABLE = "COSMOS_HTTP2_ENABLED"; + + // Config to indicate the maximum number of live connections to keep in the pool for http2 + private static final int DEFAULT_HTTP2_MAX_CONNECTION_POOL_SIZE = 1000; + private static final String HTTP2_MAX_CONNECTION_POOL_SIZE = "COSMOS.HTTP2_MAX_CONNECTION_POOL_SIZE"; + private static final String HTTP2_MAX_CONNECTION_POOL_SIZE_VARIABLE = "COSMOS_HTTP2_MAX_CONNECTION_POOL_SIZE"; + + // Config to indicate the minimum number of live connections to keep in the pool for http2 + private static final int DEFAULT_HTTP2_MIN_CONNECTION_POOL_SIZE = 1; + private static final String HTTP2_MIN_CONNECTION_POOL_SIZE = "COSMOS.HTTP2_MIN_CONNECTION_POOL_SIZE"; + private static final String HTTP2_MIN_CONNECTION_POOL_SIZE_VARIABLE = "COSMOS_HTTP2_MIN_CONNECTION_POOL_SIZE"; + + // Config to indicate the maximum number of the concurrent streams that can be opened to the remote peer for http2 + private static final int DEFAULT_HTTP2_MAX_CONCURRENT_STREAMS = 30; + private static final String HTTP2_MAX_CONCURRENT_STREAMS = "COSMOS.HTTP2_MAX_CONCURRENT_STREAMS"; + private static final String HTTP2_MAX_CONCURRENT_STREAMS_VARIABLE = "COSMOS_HTTP2_MAX_CONCURRENT_STREAMS"; public static int getCPUCnt() { return CPU_CNT; } - private SslContext sslContextInit(boolean serverCertVerificationDisabled) { + private SslContext sslContextInit(boolean serverCertVerificationDisabled, boolean http2Enabled) { try { SslContextBuilder sslContextBuilder = SslContextBuilder .forClient() .sslProvider(SslContext.defaultClientProvider()); + if (serverCertVerificationDisabled) { sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); // disable cert verification } + if (http2Enabled) { + sslContextBuilder + .applicationProtocolConfig( + new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ApplicationProtocolNames.HTTP_1_1 + ) + ); + } return sslContextBuilder.build(); } catch (SSLException sslException) { logger.error("Fatal error cannot instantiate ssl context due to {}", sslException.getMessage(), sslException); @@ -308,8 +336,8 @@ private SslContext sslContextInit(boolean serverCertVerificationDisabled) { } } - public SslContext getSslContext(boolean serverCertValidationDisabled) { - return serverCertValidationDisabled ? this.sslContextWithCertValidationDisabled : this.sslContext; + public SslContext getSslContext(boolean serverCertValidationDisabled, boolean http2Enabled) { + return sslContextInit(serverCertValidationDisabled, http2Enabled); } public Protocol getProtocol() { @@ -882,6 +910,46 @@ public static boolean isHttpConnectionWithoutTLSAllowed() { return Boolean.parseBoolean(httpForEmulatorAllowed); } + public static boolean isHttp2Enabled() { + String httpEnabledConfig = System.getProperty( + HTTP2_ENABLED, + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_ENABLED_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_ENABLED))); + + return Boolean.parseBoolean(httpEnabledConfig); + } + + public static int getHttp2MaxConnectionPoolSize() { + String http2MaxConnectionPoolSize = System.getProperty( + HTTP2_MAX_CONNECTION_POOL_SIZE, + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_MAX_CONNECTION_POOL_SIZE_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_MAX_CONNECTION_POOL_SIZE))); + + return Integer.parseInt(http2MaxConnectionPoolSize); + } + + public static int getHttp2MinConnectionPoolSize() { + String http2MinConnectionPoolSize = System.getProperty( + HTTP2_MIN_CONNECTION_POOL_SIZE, + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_MIN_CONNECTION_POOL_SIZE_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_MIN_CONNECTION_POOL_SIZE))); + + return Integer.parseInt(http2MinConnectionPoolSize); + } + + public static int getHttp2MaxConcurrentStreams() { + String http2MaxConcurrentStreams = System.getProperty( + HTTP2_MAX_CONCURRENT_STREAMS, + firstNonNull( + emptyToNull(System.getenv().get(HTTP2_MAX_CONCURRENT_STREAMS_VARIABLE)), + String.valueOf(DEFAULT_HTTP2_MAX_CONCURRENT_STREAMS))); + + return Integer.parseInt(http2MaxConcurrentStreams); + } + public static boolean isEmulatorServerCertValidationDisabled() { String certVerificationDisabledConfig = System.getProperty( EMULATOR_SERVER_CERTIFICATE_VALIDATION_DISABLED, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index b744c01244ff0..451bd1991227e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -42,6 +42,7 @@ import com.azure.cosmos.implementation.directconnectivity.StoreClientFactory; import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; +import com.azure.cosmos.implementation.http.Http2ConnectionConfig; import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.http.HttpClientConfig; import com.azure.cosmos.implementation.http.HttpHeaders; @@ -831,7 +832,8 @@ private HttpClient httpClient() { .withPoolSize(this.connectionPolicy.getMaxConnectionPoolSize()) .withProxy(this.connectionPolicy.getProxy()) .withNetworkRequestTimeout(this.connectionPolicy.getHttpNetworkRequestTimeout()) - .withServerCertValidationDisabled(this.connectionPolicy.isServerCertValidationDisabled()); + .withServerCertValidationDisabled(this.connectionPolicy.isServerCertValidationDisabled()) + .withHttp2Config(new Http2ConnectionConfig()); if (connectionSharingAcrossClientsEnabled) { return SharedGatewayHttpClient.getOrCreateInstance(httpClientConfig, diagnosticsClientConfig); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java index 38b8c4ec91dab..614115a6d63e6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java @@ -139,7 +139,7 @@ public RntbdTransportClient( final GlobalEndpointManager globalEndpointManager) { this( new Options.Builder(connectionPolicy).userAgent(userAgent).build(), - configs.getSslContext(connectionPolicy.isServerCertValidationDisabled()), + configs.getSslContext(connectionPolicy.isServerCertValidationDisabled(), false), addressResolver, clientTelemetry, globalEndpointManager); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/SharedTransportClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/SharedTransportClient.java index 5c9f02093d8f4..f2975a37d271c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/SharedTransportClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/SharedTransportClient.java @@ -82,7 +82,7 @@ private SharedTransportClient( this.transportClient = new RntbdTransportClient( rntbdOptions, - configs.getSslContext(connectionPolicy.isServerCertValidationDisabled()), + configs.getSslContext(connectionPolicy.isServerCertValidationDisabled(), false), addressResolver, clientTelemetry, globalEndpointManager); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreClientFactory.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreClientFactory.java index d180f34f4d99a..16f5124a18954 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreClientFactory.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreClientFactory.java @@ -57,7 +57,7 @@ public StoreClientFactory( this.transportClient = new RntbdTransportClient( rntbdOptions, - configs.getSslContext(connectionPolicy.isServerCertValidationDisabled()), + configs.getSslContext(connectionPolicy.isServerCertValidationDisabled(), false), addressResolver, clientTelemetry, globalEndpointManager); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ConnectionConfig.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ConnectionConfig.java new file mode 100644 index 0000000000000..f3d005804fd9a --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ConnectionConfig.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.http; + +import com.azure.cosmos.implementation.Configs; + +// [TODO Http2]: when adding public API, making this class public, add setter method +public class Http2ConnectionConfig { + private int maxConnectionPoolSize; + private int minConnectionPoolSize; + private int maxConcurrentStreams; + private boolean enabled; + + public Http2ConnectionConfig() { + this.maxConnectionPoolSize = Configs.getHttp2MaxConnectionPoolSize(); + this.minConnectionPoolSize = Configs.getHttp2MinConnectionPoolSize(); + this.maxConcurrentStreams = Configs.getHttp2MaxConcurrentStreams(); + this.enabled = Configs.isHttp2Enabled(); + } + + public int getMaxConnectionPoolSize() { + return maxConnectionPoolSize; + } + + public int getMinConnectionPoolSize() { + return minConnectionPoolSize; + } + + public int getMaxConcurrentStreams() { + return maxConcurrentStreams; + } + + public boolean isEnabled() { + return enabled; + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java new file mode 100644 index 0000000000000..af63e97285f05 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.http; + +import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2HeadersFrame; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Http2ResponseHeaderCleanerHandler extends ChannelInboundHandlerAdapter { + + private static final Logger logger = LoggerFactory.getLogger(Http2ResponseHeaderCleanerHandler.class); + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + + if (msg instanceof Http2HeadersFrame) { + Http2HeadersFrame headersFrame = (Http2HeadersFrame) msg; + Http2Headers headers = headersFrame.headers(); + + headers.forEach(entry -> { + CharSequence key = entry.getKey(); + CharSequence value = entry.getValue(); + + // Based on the tests, only 'x-ms-serviceversion' header has extra value, + // so only check this specific header here + if (StringUtils.equalsIgnoreCase(key, HttpConstants.HttpHeaders.SERVER_VERSION)) { + // Check for leading whitespace or other prohibited characters + if (StringUtils.isNotEmpty(value) && (value.charAt(0) == ' ' || value.charAt(value.length() - 1) == ' ')) { + // Clean up the header value by trimming or handling as needed + logger.trace("There are extra white space for key {} with value {}", key, value); + + // TODO[Http2]: for now just trim the spaces, explore other options for example escape the whitespace + headers.set(key, value.toString().trim()); + } + } + }); + } + + // Pass the message to the next handler in the pipeline + super.channelRead(ctx, msg); + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java index 2f8747c2df0ec..22eb30e7174dc 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java @@ -3,6 +3,7 @@ package com.azure.cosmos.implementation.http; import reactor.core.publisher.Mono; +import reactor.netty.http.client.Http2AllocationStrategy; import reactor.netty.resources.ConnectionProvider; import java.time.Duration; @@ -45,6 +46,16 @@ static HttpClient createFixed(HttpClientConfig httpClientConfig) { fixedConnectionProviderBuilder.pendingAcquireTimeout(httpClientConfig.getConnectionAcquireTimeout()); fixedConnectionProviderBuilder.maxIdleTime(httpClientConfig.getMaxIdleConnectionTimeout()); + if (httpClientConfig.getHttp2Config().isEnabled()) { + fixedConnectionProviderBuilder.allocationStrategy( + Http2AllocationStrategy.builder() + .maxConnections(httpClientConfig.getHttp2Config().getMaxConnectionPoolSize()) + .minConnections(httpClientConfig.getHttp2Config().getMinConnectionPoolSize()) + .maxConcurrentStreams(httpClientConfig.getHttp2Config().getMaxConcurrentStreams()) + .build() + ); + } + return ReactorNettyClient.createWithConnectionProvider(fixedConnectionProviderBuilder.build(), httpClientConfig); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java index 5b0946d2c9940..d13b8d7317aee 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java @@ -27,9 +27,11 @@ public class HttpClientConfig { private ProxyOptions proxy; private boolean connectionKeepAlive = true; private boolean serverCertValidationDisabled = false; + private Http2ConnectionConfig http2ConnectionConfig; public HttpClientConfig(Configs configs) { this.configs = configs; + this.http2ConnectionConfig = new Http2ConnectionConfig(); } public HttpClientConfig withMaxHeaderSize(int maxHeaderSize) { @@ -97,6 +99,11 @@ public HttpClientConfig withServerCertValidationDisabled(boolean serverCertValid return this; } + public HttpClientConfig withHttp2Config(Http2ConnectionConfig http2ConnectionConfig) { + this.http2ConnectionConfig = http2ConnectionConfig; + return this; + } + public Configs getConfigs() { return configs; } @@ -153,6 +160,10 @@ public boolean isServerCertValidationDisabled() { return serverCertValidationDisabled; } + public Http2ConnectionConfig getHttp2Config() { + return http2ConnectionConfig; + } + public String toDiagnosticsString() { return String.format("(cps:%s, nrto:%s, icto:%s, cto:%s, p:%s)", maxPoolSize, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index bdfcc6548eb6d..ddd30baa9256e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -5,6 +5,7 @@ import com.azure.cosmos.implementation.Configs; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.logging.LogLevel; import io.netty.resolver.DefaultAddressResolverGroup; @@ -17,6 +18,7 @@ import reactor.netty.Connection; import reactor.netty.ConnectionObserver; import reactor.netty.NettyOutbound; +import reactor.netty.http.HttpProtocol; import reactor.netty.http.client.HttpClientRequest; import reactor.netty.http.client.HttpClientResponse; import reactor.netty.http.client.HttpClientState; @@ -136,13 +138,39 @@ private void configureChannelPipelineHandlers() { this.httpClient = this.httpClient .secure(sslContextSpec -> - sslContextSpec.sslContext(configs.getSslContext(httpClientConfig.isServerCertValidationDisabled()))) + sslContextSpec.sslContext( + configs.getSslContext( + httpClientConfig.isServerCertValidationDisabled(), + false))) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) this.httpClientConfig.getConnectionAcquireTimeout().toMillis()) .httpResponseDecoder(httpResponseDecoderSpec -> httpResponseDecoderSpec.maxInitialLineLength(this.httpClientConfig.getMaxInitialLineLength()) .maxHeaderSize(this.httpClientConfig.getMaxHeaderSize()) .maxChunkSize(this.httpClientConfig.getMaxChunkSize()) .validateHeaders(true)); + + if (httpClientConfig.getHttp2Config().isEnabled()) { + this.httpClient = this.httpClient + .secure(sslContextSpec -> + sslContextSpec.sslContext( + configs.getSslContext( + httpClientConfig.isServerCertValidationDisabled(), + httpClientConfig.getHttp2Config().isEnabled() + ))) + .protocol(HttpProtocol.H2, HttpProtocol.HTTP11) + .doOnConnected((connection -> { + // The response header clean up pipeline is being added due to an error getting when calling gateway: + // java.lang.IllegalArgumentException: a header value contains prohibited character 0x20 at index 0 for 'x-ms-serviceversion', there is whitespace in the front of the value. + // validateHeaders(false) does not work for http2 + ChannelPipeline channelPipeline = connection.channel().pipeline(); + if (channelPipeline.get("reactor.left.httpCodec") != null) { + channelPipeline.addAfter( + "reactor.left.httpCodec", + "customHeaderCleaner", + new Http2ResponseHeaderCleanerHandler()); + } + })); + } } @Override diff --git a/sdk/cosmos/live-http2-platform-matrix.json b/sdk/cosmos/live-http2-platform-matrix.json new file mode 100644 index 0000000000000..abb531288c40c --- /dev/null +++ b/sdk/cosmos/live-http2-platform-matrix.json @@ -0,0 +1,27 @@ +{ + "displayNames": { + "-Pfast": "Fast", + "-Pquery": "Query", + "-Pcircuit-breaker-misc-gateway": "CircuitBreakerMiscGateway", + "Session": "", + "ubuntu": "", + "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }": "http2" + }, + "include": [ + { + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "ACCOUNT_CONSISTENCY": "Session", + "ArmConfig": { + "MultiMaster_MultiRegion": { + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }", + "PREFERRED_LOCATIONS": "[\"East US 2\"]" + } + }, + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ "-Pquery", "-Pcircuit-breaker-misc-gateway", "-Pfast" ], + "Agent": { + "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } + } + } + ] +} diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index b285f1216c060..66f760673e511 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -37,6 +37,40 @@ extends: - name: AdditionalArgs value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml + parameters: + TestName: 'Cosmos_Live_Test_Http2' + CloudConfig: + Public: + ServiceConnection: azure-sdk-tests-cosmos + MatrixConfigs: + - Name: Cosmos_live_test_http2 + Path: sdk/cosmos/live-http2-platform-matrix.json + Selection: all + GenerateVMJobs: true + MatrixReplace: + - .*Version=1.21/1.17 + ServiceDirectory: cosmos + Artifacts: + - name: azure-cosmos + groupId: com.azure + safeName: azurecosmos + AdditionalModules: + - name: azure-cosmos-tests + groupId: com.azure + - name: azure-cosmos-benchmark + groupId: com.azure + TimeoutInMinutes: 210 + MaxParallel: 20 + PreSteps: + - template: /eng/pipelines/templates/steps/install-reporting-tools.yml + TestGoals: 'verify' + TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' + TestResultsFiles: '**/junitreports/TEST-*.xml' + AdditionalVariables: + - name: AdditionalArgs + value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DCOSMOS.HTTP2_ENABLED=true' + - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: TestName: 'Spring_Data_Cosmos_Integration'