allowUsingHttp2

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java

@@ -5,8 +5,12 @@
 import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
 import com.azure.cosmos.implementation.circuitBreaker.PartitionLevelCircuitBreakerConfig;
 import com.azure.cosmos.implementation.directconnectivity.Protocol;
+import io.netty.handler.codec.http2.Http2SecurityUtil;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.ApplicationProtocolNames;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SupportedCipherSuiteFilter;
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,7 +36,7 @@ public class Configs {
     public static final String SPECULATION_THRESHOLD = "COSMOS_SPECULATION_THRESHOLD";
     public static final String SPECULATION_THRESHOLD_STEP = "COSMOS_SPECULATION_THRESHOLD_STEP";
     private final SslContext sslContext;
-
+    private final SslContext sslContextWithHttp2Enabled;
     // The names we use are consistent with the:
     // * Azure environment variable naming conventions documented at https://azure.github.io/azure-sdk/java_implementation.html and
     // * Java property naming conventions as illustrated by the name/value pairs returned by System.getProperties.
@@ -261,24 +265,43 @@ public class Configs {
     private static final String INSECURE_EMULATOR_CONNECTION_ALLOWED = "COSMOS.INSECURE_EMULATOR_CONNECTION_ALLOWED";
     private static final String INSECURE_EMULATOR_CONNECTION_ALLOWED_VARIABLE = "COSMOS_INSECURE_EMULATOR_CONNECTION_ALLOWED";
 
+    // Flag to indicate whether enabled http2 for gateway, Please do not use it, only for internal testing purpose
+    private static final boolean DEFAULT_HTTP2_ENABLED = false;
+    private static final String HTTP2_ENABLED = "COSMOS.HTTP2_ENABLED";
+    private static final String HTTP2_ENABLED_VARIABLE = "COSMOS_HTTP2_ENABLED";
+
     public Configs() {
-        this.sslContext = sslContextInit();
+        this.sslContext = sslContextInit(false);
+        this.sslContextWithHttp2Enabled = sslContextInit(true);
     }
 
     public static int getCPUCnt() {
         return CPU_CNT;
     }
 
-    private SslContext sslContextInit() {
+    private SslContext sslContextInit(boolean http2Enabled) {
         try {
             SslContextBuilder sslContextBuilder =
                 SslContextBuilder
                     .forClient()
                     .sslProvider(SslContext.defaultClientProvider());
+
             if (isInsecureEmulatorConnectionAllowed()) {
                 sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); // disable cert verification
             }
 
+            if (http2Enabled) {
+                sslContextBuilder
+                    .applicationProtocolConfig(
+                        new ApplicationProtocolConfig(
+                            ApplicationProtocolConfig.Protocol.ALPN,
+                            ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
+                            ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
+                            ApplicationProtocolNames.HTTP_2,
+                            ApplicationProtocolNames.HTTP_1_1
+                        )
+                    );
+            }
             return sslContextBuilder.build();
         } catch (SSLException sslException) {
             logger.error("Fatal error cannot instantiate ssl context due to {}", sslException.getMessage(), sslException);
@@ -290,6 +313,10 @@ public SslContext getSslContext() {
         return this.sslContext;
     }
 
+    public SslContext getSslContextWithHttp2Enabled() {
+        return this.sslContextWithHttp2Enabled;
+    }
+
     public Protocol getProtocol() {
         String protocol = System.getProperty(PROTOCOL_PROPERTY, firstNonNull(
             emptyToNull(System.getenv().get(PROTOCOL_ENVIRONMENT_VARIABLE)),
@@ -835,4 +862,14 @@ public static boolean isInsecureEmulatorConnectionAllowed() {
 
         return Boolean.parseBoolean(httpForEmulatorAllowed);
     }
+
+    public static boolean isHttp2Enabled() {
+        String httpEnabledConfig = System.getProperty(
+            HTTP2_ENABLED,
+            firstNonNull(
+                emptyToNull(System.getenv().get(HTTP2_ENABLED_VARIABLE)),
+                String.valueOf(DEFAULT_HTTP2_ENABLED)));
+
+        return Boolean.parseBoolean(httpEnabledConfig);
+    }
 }

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java

@@ -95,7 +95,6 @@
 import reactor.core.publisher.SignalType;
 import reactor.util.concurrent.Queues;
 import reactor.util.function.Tuple2;
-import reactor.util.retry.Retry;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -672,9 +671,6 @@ private void updateGatewayProxy() {
 
     public void init(CosmosClientMetadataCachesSnapshot metadataCachesSnapshot, Function<HttpClient, HttpClient> httpClientInterceptor) {
         try {
-            // TODO: add support for openAsync
-            // https://msdata.visualstudio.com/CosmosDB/_workitems/edit/332589
-
             this.httpClientInterceptor = httpClientInterceptor;
             if (httpClientInterceptor != null) {
                 this.reactorHttpClient = httpClientInterceptor.apply(httpClient());
@@ -835,7 +831,8 @@ private HttpClient httpClient() {
             .withMaxIdleConnectionTimeout(this.connectionPolicy.getIdleHttpConnectionTimeout())
             .withPoolSize(this.connectionPolicy.getMaxConnectionPoolSize())
             .withProxy(this.connectionPolicy.getProxy())
-            .withNetworkRequestTimeout(this.connectionPolicy.getHttpNetworkRequestTimeout());
+            .withNetworkRequestTimeout(this.connectionPolicy.getHttpNetworkRequestTimeout())
+            .withHttp2Enabled(Configs.isHttp2Enabled());
 
         if (connectionSharingAcrossClientsEnabled) {
             return SharedGatewayHttpClient.getOrCreateInstance(httpClientConfig, diagnosticsClientConfig);

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.http;
+
+import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.codec.http2.Http2Headers;
+import io.netty.handler.codec.http2.Http2HeadersFrame;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Http2ResponseHeaderCleanerHandler extends ChannelInboundHandlerAdapter {
+
+    private static final Logger logger = LoggerFactory.getLogger(Http2ResponseHeaderCleanerHandler.class);
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+
+        if (msg instanceof Http2HeadersFrame) {
+            Http2HeadersFrame headersFrame = (Http2HeadersFrame) msg;
+            Http2Headers headers = headersFrame.headers();
+
+            headers.forEach(entry -> {
+                CharSequence key = entry.getKey();
+                CharSequence value = entry.getValue();
+
+                // Check for leading whitespace or other prohibited characters
+                if (StringUtils.isNotEmpty(value) && (value.charAt(0) == ' ' || value.charAt(value.length() - 1) == ' ')) {
+                    // Clean up the header value by trimming or handling as needed
+                    logger.warn("There are extra white space for key {} with value {}", key, value);
+
+                    // TODO[Http2]: for now just trim the spaces, explore other options for example escape the whitespace
+                    headers.set(key, value.toString().trim());
+                }
+            });
+        }
+
+        // Pass the message to the next handler in the pipeline
+        super.channelRead(ctx, msg);
+    }
+}

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClient.java

@@ -59,6 +59,8 @@ static HttpClient createFixed(HttpClientConfig httpClientConfig) {
         fixedConnectionProviderBuilder.pendingAcquireTimeout(connectionAcquireTimeout);
         fixedConnectionProviderBuilder.maxIdleTime(maxIdleConnectionTimeoutInMillis);
 
+        // TODO[Http2]: config Http2AllocationStrategy (maxConnections & maxConcurrentStreams)
+
         return ReactorNettyClient.createWithConnectionProvider(fixedConnectionProviderBuilder.build(),
             httpClientConfig);
     }

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/HttpClientConfig.java

@@ -21,6 +21,7 @@ public class HttpClientConfig {
     private Duration networkRequestTimeout;
     private ProxyOptions proxy;
     private boolean connectionKeepAlive = true;
+    private boolean http2Enabled;
 
     public HttpClientConfig(Configs configs) {
         this.configs = configs;
@@ -51,6 +52,11 @@ public HttpClientConfig withConnectionKeepAlive(boolean connectionKeepAlive) {
         return this;
     }
 
+    public HttpClientConfig withHttp2Enabled(boolean http2Enabled) {
+        this.http2Enabled = http2Enabled;
+        return this;
+    }
+
     public Configs getConfigs() {
         return configs;
     }
@@ -75,6 +81,10 @@ public boolean isConnectionKeepAlive() {
         return connectionKeepAlive;
     }
 
+    public boolean isHttp2Enabled() {
+        return http2Enabled;
+    }
+
     //  TODO(kuthapar): Do we really need to use Strings.lenientFormat() here?
     //  Even the documentation of this API suggests to use String.format or just string appends if possible.
     public String toDiagnosticsString() {

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java

@@ -5,8 +5,10 @@
 import com.azure.cosmos.implementation.Configs;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.logging.LogLevel;
+import io.netty.handler.ssl.SslHandler;
 import io.netty.resolver.DefaultAddressResolverGroup;
 import org.reactivestreams.Publisher;
 import org.reactivestreams.Subscription;
@@ -17,6 +19,7 @@
 import reactor.netty.Connection;
 import reactor.netty.ConnectionObserver;
 import reactor.netty.NettyOutbound;
+import reactor.netty.http.HttpProtocol;
 import reactor.netty.http.client.HttpClientRequest;
 import reactor.netty.http.client.HttpClientResponse;
 import reactor.netty.http.client.HttpClientState;
@@ -76,7 +79,7 @@ public static ReactorNettyClient create(HttpClientConfig httpClientConfig) {
             .newConnection()
             .observe(getConnectionObserver())
             .resolver(DefaultAddressResolverGroup.INSTANCE);
-        reactorNettyClient.configureChannelPipelineHandlers();
+        reactorNettyClient.configureChannelPipelineHandlers(httpClientConfig.isHttp2Enabled());
         attemptToWarmupHttpClient(reactorNettyClient);
         return reactorNettyClient;
     }
@@ -92,7 +95,7 @@ public static ReactorNettyClient createWithConnectionProvider(ConnectionProvider
             .create(connectionProvider)
             .observe(getConnectionObserver())
             .resolver(DefaultAddressResolverGroup.INSTANCE);
-        reactorNettyClient.configureChannelPipelineHandlers();
+        reactorNettyClient.configureChannelPipelineHandlers(httpClientConfig.isHttp2Enabled());
         attemptToWarmupHttpClient(reactorNettyClient);
         return reactorNettyClient;
     }
@@ -117,7 +120,7 @@ private static void attemptToWarmupHttpClient(ReactorNettyClient reactorNettyCli
         }
     }
 
-    private void configureChannelPipelineHandlers() {
+    private void configureChannelPipelineHandlers(boolean http2Enabled) {
         Configs configs = this.httpClientConfig.getConfigs();
 
         if (this.httpClientConfig.getProxy() != null) {
@@ -139,6 +142,24 @@ private void configureChannelPipelineHandlers() {
                     .maxHeaderSize(configs.getMaxHttpHeaderSize())
                     .maxChunkSize(configs.getMaxHttpChunkSize())
                     .validateHeaders(true));
+
+        if (http2Enabled) {
+            this.httpClient = this.httpClient
+                .secure(sslContextSpec -> sslContextSpec.sslContext(configs.getSslContextWithHttp2Enabled()))
+                .protocol(HttpProtocol.H2, HttpProtocol.HTTP11)
+                .doOnConnected((connection -> {
+                    // The response header clean up pipeline is being added due to an error getting when calling gateway:
+                    // java.lang.IllegalArgumentException: a header value contains prohibited character 0x20 at index 0 for 'x-ms-serviceversion', there is whitespace in the front of the value.
+                    // validateHeaders(false) does not work for http2
+                    ChannelPipeline channelPipeline = connection.channel().pipeline();
+                    if (channelPipeline.get("reactor.left.httpCodec") != null) {
+                        channelPipeline.addAfter(
+                            "reactor.left.httpCodec",
+                            "customHeaderCleaner",
+                            new Http2ResponseHeaderCleanerHandler());
+                    }
+                }));
+        }
     }
 
     @Override

sdk/cosmos/live-http2-platform-matrix.json

@@ -0,0 +1,27 @@
+{
+  "displayNames": {
+    "-Pfast": "Fast",
+    "-Pquery": "Query",
+    "-Pcircuit-breaker-misc-gateway": "CircuitBreakerMiscGateway",
+    "Session": "",
+    "ubuntu": "",
+    "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }": ""
+  },
+  "include": [
+    {
+      "DESIRED_CONSISTENCIES": "[\"Session\"]",
+      "ACCOUNT_CONSISTENCY": "Session",
+      "ArmConfig": {
+        "MultiMaster_MultiRegion": {
+          "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }",
+          "PREFERRED_LOCATIONS": "[\"East US 2\"]"
+        }
+      },
+      "PROTOCOLS": "[\"Tcp\"]",
+      "ProfileFlag": [ "-Pquery", "-Pcircuit-breaker-misc-gateway", "-Pfast" ],
+      "Agent": {
+        "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
+      }
+    }
+  ]
+}

sdk/cosmos/tests.yml

@@ -37,6 +37,40 @@ extends:
             - name: AdditionalArgs
               value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)'
 
+      - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml
+        parameters:
+          TestName: 'Cosmos_Live_Test_Http2'
+          CloudConfig:
+            Public:
+              ServiceConnection: azure-sdk-tests-cosmos
+          MatrixConfigs:
+            - Name: Cosmos_live_test_http2
+              Path: sdk/cosmos/live-http2-platform-matrix.json
+              Selection: all
+              GenerateVMJobs: true
+          MatrixReplace:
+            - .*Version=1.21/1.17
+          ServiceDirectory: cosmos
+          Artifacts:
+            - name: azure-cosmos
+              groupId: com.azure
+              safeName: azurecosmos
+          AdditionalModules:
+            - name: azure-cosmos-tests
+              groupId: com.azure
+            - name: azure-cosmos-benchmark
+              groupId: com.azure
+          TimeoutInMinutes: 210
+          MaxParallel: 20
+          PreSteps:
+            - template: /eng/pipelines/templates/steps/install-reporting-tools.yml
+          TestGoals: 'verify'
+          TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false'
+          TestResultsFiles: '**/junitreports/TEST-*.xml'
+          AdditionalVariables:
+            - name: AdditionalArgs
+              value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DCOSMOS.HTTP2_ENABLED=true'
+
       - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml
         parameters:
           TestName: 'Spring_Data_Cosmos_Integration'