Stop partition recovery flow when a client instance is closed. (#41486)

* Push down collection and partition key range resolution. * Tweaking threshold behavior. * Fixing tests. * Fixing tests. * Fixing tests. * Fixing tests. * Perform collectionLink normalization. * Fix CI pipeline. * Reacting to review comments. * Updated CHANGELOG.md. * Force circuit breaking to be enabled. * Increase error thresholds. * Force circuit breaker for certain tests. * Scope live test matrix. * Scope live test matrix. * Scope live test matrix. * Close globalPartitionEndpointManagerForCircuitBreaker. * Close globalPartitionEndpointManagerForCircuitBreaker. * Use non-static scheduler. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Test multi-region + circuit-breaker job. * Scrubbing off Java 21 emulator targeting Spring emulator test. * Refactoring. * Reacting to review comments. * Attempt at fixing live tests pipeline. * Reacting to review comments. * Reacting to review comments. * Reacting to review comments. * Revert some live test pipeline changes. * Updated CHANGELOG.md.
Azure · Aug 22, 2024 · a7cb0c5 · a7cb0c5
1 parent 795e96d
commit a7cb0c5
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 41 deletions.
diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json
@@ -18,19 +18,6 @@
         "JavaTestVersion": "1.17",
         "AdditionalArgs": "-DargLine=\"-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true\""
       },
-      "Spring Emulator Only Integration Tests - Java 21": {
-        "ProfileFlag": "-Pintegration-test-emulator",
-        "JavaTestVersion": "1.21",
-        "ACCOUNT_HOST": "https://localhost:8081/",
-        "ACCOUNT_KEY": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
-        "SECONDARY_ACCOUNT_KEY": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
-        "NEW_ACCOUNT_HOST": "https://localhost:8081/",
-        "NEW_ACCOUNT_KEY": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
-        "NEW_SECONDARY_ACCOUNT_KEY": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
-        "TestFromSource": true,
-        "AdditionalArgs": "-Dspring-e2e -DargLine=-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
-        "Language": "Spring"
-      },
       "Spring Emulator Only Integration Tests - Java 17": {
           "ProfileFlag": "-Pintegration-test-emulator",
           "JavaTestVersion": "1.17",

diff --git a/...src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java b/...src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java
@@ -71,6 +71,7 @@ public static Object[][] operationTypeProvider() {
             { FaultInjectionOperationType.READ_ITEM, OperationType.Read },
             { FaultInjectionOperationType.REPLACE_ITEM, OperationType.Replace },
             { FaultInjectionOperationType.CREATE_ITEM, OperationType.Create },
+            { FaultInjectionOperationType.UPSERT_ITEM, OperationType.Upsert },
             { FaultInjectionOperationType.DELETE_ITEM, OperationType.Delete },
             { FaultInjectionOperationType.QUERY_ITEM, OperationType.Query },
             { FaultInjectionOperationType.PATCH_ITEM, OperationType.Patch }

diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md
@@ -7,6 +7,7 @@
 #### Breaking Changes
 
 #### Bugs Fixed
+* Fixed a direct buffer memory leak due to not explicitly stopping the partition recovery flow in per-partition circuit breaker. - See [PR 41486](https://github.com/Azure/azure-sdk-for-java/pull/41486)
 
 #### Other Changes
 
@@ -15,7 +16,7 @@
 #### Bugs Fixed
 * Fixed an eager prefetch issue for order by queries to prevent unnecessary round trips. - See [PR 41348](https://github.com/Azure/azure-sdk-for-java/pull/41348)
 * Fixed an issue to not fail fast for metadata resource resolution when faults are injected for Gateway routed operations. - See [PR 41428](https://github.com/Azure/azure-sdk-for-java/pull/41428)
-* Fixed an issue to adhere with exception tolerance thresholds for consecutive read and write failures with circuit breaker. - See [PR 41248](https://github.com/Azure/azure-sdk-for-java/pull/41428)
+* Fixed an issue to adhere with exception tolerance thresholds for consecutive read and write failures with circuit breaker. - See [PR 41428](https://github.com/Azure/azure-sdk-for-java/pull/41428)
 * Fixed excessive retries bug when it has been identified that operations through a closed `CosmosClient` [or] `CosmosAsyncClient` are executed. - See [PR 41364](https://github.com/Azure/azure-sdk-for-java/pull/41364)
 
 #### Other Changes

diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java
@@ -18,6 +18,7 @@
 import com.azure.cosmos.implementation.apachecommons.collections.list.UnmodifiableList;
 import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
 import com.azure.cosmos.implementation.apachecommons.lang.time.StopWatch;
+import com.azure.cosmos.implementation.circuitBreaker.PartitionLevelCircuitBreakerConfig;
 import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry;
 import com.azure.cosmos.implementation.guava25.base.Preconditions;
 import com.azure.cosmos.implementation.routing.LocationHelper;
@@ -1177,6 +1178,11 @@ public CosmosAsyncClient buildAsyncClient() {
     CosmosAsyncClient buildAsyncClient(boolean logStartupInfo) {
         StopWatch stopwatch = new StopWatch();
         stopwatch.start();
+
+        if (Configs.shouldOptInDefaultCircuitBreakerConfig()) {
+            System.setProperty("COSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_CONFIG", "{\"isPartitionLevelCircuitBreakerEnabled\": true}");
+        }
+
         this.resetSessionCapturingType();
         validateConfig();
         buildConnectionPolicy();
@@ -1212,6 +1218,11 @@ CosmosAsyncClient buildAsyncClient(boolean logStartupInfo) {
     public CosmosClient buildClient() {
         StopWatch stopwatch = new StopWatch();
         stopwatch.start();
+
+        if (Configs.shouldOptInDefaultCircuitBreakerConfig()) {
+            System.setProperty("COSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_CONFIG", "{\"isPartitionLevelCircuitBreakerEnabled\": true}");
+        }
+
         this.resetSessionCapturingType();
         validateConfig();
         buildConnectionPolicy();

diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java
@@ -245,6 +245,10 @@ public class Configs {
     private static final boolean DEFAULT_SHOULD_LOG_INCORRECTLY_MAPPED_SESSION_TOKEN = true;
     private static final String SHOULD_LOG_INCORRECTLY_MAPPED_SESSION_TOKEN = "COSMOS.SHOULD_LOG_INCORRECTLY_MAPPED_USER_SESSION_TOKEN";
 
+    private static final boolean DEFAULT_PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN = false;
+    private static final String PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN = "COSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN";
+
+
     public Configs() {
         this.sslContext = sslContextInit();
     }
@@ -657,6 +661,18 @@ public static boolean shouldLogIncorrectlyMappedSessionToken() {
         return Boolean.parseBoolean(shouldSystemExit);
     }
 
+    public static boolean shouldOptInDefaultCircuitBreakerConfig() {
+
+        String shouldOptInDefaultPartitionLevelCircuitBreakerConfig =
+            System.getProperty(
+                PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN,
+                firstNonNull(
+                    emptyToNull(System.getenv().get(PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN)),
+                    String.valueOf(DEFAULT_PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN)));
+
+        return Boolean.parseBoolean(shouldOptInDefaultPartitionLevelCircuitBreakerConfig);
+    }
+
     public static CosmosMicrometerMetricsConfig getMetricsConfig() {
         String metricsConfig =
             System.getProperty(

diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/CosmosSchedulers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/CosmosSchedulers.java
@@ -86,9 +86,4 @@ public class CosmosSchedulers {
         TTL_FOR_SCHEDULER_WORKER_IN_SECONDS,
         true
     );
-
-    public final static Scheduler PARTITION_AVAILABILITY_STALENESS_CHECK_SINGLE = Schedulers.newSingle(
-      "partition-availability-staleness-check",
-      true
-    );
 }
diff --git a/...smos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/...smos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java
@@ -5748,6 +5748,12 @@ public void close() {
         if (!closed.getAndSet(true)) {
             activeClientsCnt.decrementAndGet();
             logger.info("Shutting down ...");
+
+            if (this.globalPartitionEndpointManagerForCircuitBreaker != null) {
+                logger.info("Closing globalPartitionEndpointManagerForCircuitBreaker...");
+                LifeCycleUtils.closeQuietly(this.globalPartitionEndpointManagerForCircuitBreaker);
+            }
+
             logger.info("Closing Global Endpoint Manager ...");
             LifeCycleUtils.closeQuietly(this.globalEndpointManager);
             logger.info("Closing StoreClientFactory ...");
@@ -5968,24 +5974,39 @@ public void addPartitionLevelUnavailableRegionsForRequest(
             checkNotNull(options.getPartitionKeyDefinition(), "Argument 'partitionKeyDefinition' within options cannot be null!");
             checkNotNull(collectionRoutingMap, "Argument 'collectionRoutingMap' cannot be null!");
 
+            PartitionKeyRange resolvedPartitionKeyRange = null;
+
             PartitionKeyDefinition partitionKeyDefinition = options.getPartitionKeyDefinition();
             PartitionKeyInternal partitionKeyInternal = request.getPartitionKeyInternal();
 
-            String effectivePartitionKeyString = PartitionKeyInternalHelper.getEffectivePartitionKeyString(partitionKeyInternal, partitionKeyDefinition);
-            PartitionKeyRange partitionKeyRange = collectionRoutingMap.getRangeByEffectivePartitionKey(effectivePartitionKeyString);
+            if (partitionKeyInternal != null) {
+                String effectivePartitionKeyString = PartitionKeyInternalHelper.getEffectivePartitionKeyString(partitionKeyInternal, partitionKeyDefinition);
+                resolvedPartitionKeyRange = collectionRoutingMap.getRangeByEffectivePartitionKey(effectivePartitionKeyString);
 
-            checkNotNull(partitionKeyRange, "partitionKeyRange cannot be null!");
+                // cache the effective partition key if possible - can be a bottleneck,
+                // since it is also recomputed in AddressResolver
+                request.setEffectivePartitionKey(effectivePartitionKeyString);
+            } else if (request.getPartitionKeyRangeIdentity() != null) {
+                resolvedPartitionKeyRange = collectionRoutingMap.getRangeByPartitionKeyRangeId(request.getPartitionKeyRangeIdentity().getPartitionKeyRangeId());
+            }
+
+            checkNotNull(resolvedPartitionKeyRange, "resolvedPartitionKeyRange cannot be null!");
             checkNotNull(this.globalPartitionEndpointManagerForCircuitBreaker, "globalPartitionEndpointManagerForCircuitBreaker cannot be null!");
 
+            // setting it here in case request.requestContext.resolvedPartitionKeyRange
+            // is not assigned in either GlobalAddressResolver / RxGatewayStoreModel (possible if there are Gateway timeouts)
+            // and circuit breaker also kicks in to mark a failure resolvedPartitionKeyRange (will result in NullPointerException and will
+            // help failover as well)
+            // also resolvedPartitionKeyRange will be overridden in GlobalAddressResolver / RxGatewayStoreModel irrespective
+            // so staleness is not an issue (after doing a validation of parent-child relationship b/w initial and new partitionKeyRange)
+            request.requestContext.resolvedPartitionKeyRange = resolvedPartitionKeyRange;
+
             List<String> unavailableRegionsForPartition
                 = this.globalPartitionEndpointManagerForCircuitBreaker.getUnavailableRegionsForPartitionKeyRange(
                 request.getResourceId(),
-                partitionKeyRange,
+                resolvedPartitionKeyRange,
                 request.getOperationType());
 
-            // cache the effective partition key if possible - can be a bottleneck,
-            // since it is also recomputed in AddressResolver
-            request.setEffectivePartitionKey(effectivePartitionKeyString);
             request.requestContext.setUnavailableRegionsForPartition(unavailableRegionsForPartition);
 
             // onBeforeSendRequest uses excluded regions to know the next location endpoint
@@ -6026,6 +6047,14 @@ public void addPartitionLevelUnavailableRegionsForFeedRequest(
 
         checkNotNull(resolvedPartitionKeyRange, "resolvedPartitionKeyRange cannot be null!");
 
+        // setting it here in case request.requestContext.resolvedPartitionKeyRange
+        // is not assigned in either GlobalAddressResolver / RxGatewayStoreModel (possible if there are Gateway timeouts)
+        // and circuit breaker also kicks in to mark a failure resolvedPartitionKeyRange (will result in NullPointerException and will
+        // help failover as well)
+        // also resolvedPartitionKeyRange will be overridden in GlobalAddressResolver / RxGatewayStoreModel irrespective
+        // so staleness is not an issue (after doing a validation of parent-child relationship b/w initial and new partitionKeyRange)
+        request.requestContext.resolvedPartitionKeyRange = resolvedPartitionKeyRange;
+
         if (this.globalPartitionEndpointManagerForCircuitBreaker.isPartitionLevelCircuitBreakingApplicable(request)) {
             checkNotNull(globalPartitionEndpointManagerForCircuitBreaker, "globalPartitionEndpointManagerForCircuitBreaker cannot be null!");
 

diff --git a/...cosmos/implementation/circuitBreaker/GlobalPartitionEndpointManagerForCircuitBreaker.java b/...cosmos/implementation/circuitBreaker/GlobalPartitionEndpointManagerForCircuitBreaker.java
@@ -4,7 +4,6 @@
 package com.azure.cosmos.implementation.circuitBreaker;
 
 import com.azure.cosmos.implementation.Configs;
-import com.azure.cosmos.implementation.CosmosSchedulers;
 import com.azure.cosmos.implementation.FeedOperationContextForCircuitBreaker;
 import com.azure.cosmos.implementation.GlobalEndpointManager;
 import com.azure.cosmos.implementation.ImplementationBridgeHelpers;
@@ -22,6 +21,8 @@
 import org.slf4j.LoggerFactory;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Scheduler;
+import reactor.core.scheduler.Schedulers;
 
 import java.net.URI;
 import java.time.Duration;
@@ -36,7 +37,7 @@
 
 import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull;
 
-public class GlobalPartitionEndpointManagerForCircuitBreaker {
+public class GlobalPartitionEndpointManagerForCircuitBreaker implements AutoCloseable {
 
     private static final Logger logger = LoggerFactory.getLogger(GlobalPartitionEndpointManagerForCircuitBreaker.class);
 
@@ -49,6 +50,8 @@ public class GlobalPartitionEndpointManagerForCircuitBreaker {
     private final ConsecutiveExceptionBasedCircuitBreaker consecutiveExceptionBasedCircuitBreaker;
     private final AtomicReference<GlobalAddressResolver> globalAddressResolverSnapshot;
     private final ConcurrentHashMap<URI, String> locationToRegion;
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
+    private final Scheduler partitionRecoveryScheduler = Schedulers.newSingle("partition-availability-staleness-check");
 
     public GlobalPartitionEndpointManagerForCircuitBreaker(GlobalEndpointManager globalEndpointManager) {
         this.partitionKeyRangeToLocationSpecificUnavailabilityInfo = new ConcurrentHashMap<>();
@@ -65,7 +68,7 @@ public GlobalPartitionEndpointManagerForCircuitBreaker(GlobalEndpointManager glo
 
     public void init() {
         if (this.consecutiveExceptionBasedCircuitBreaker.isPartitionLevelCircuitBreakerEnabled()) {
-            this.updateStaleLocationInfo().subscribeOn(CosmosSchedulers.PARTITION_AVAILABILITY_STALENESS_CHECK_SINGLE).subscribe();
+            this.updateStaleLocationInfo().subscribeOn(this.partitionRecoveryScheduler).subscribe();
         }
     }
 
@@ -86,8 +89,6 @@ public void handleLocationExceptionForPartitionKeyRange(RxDocumentServiceRequest
         AtomicBoolean isFailoverPossible = new AtomicBoolean(true);
         AtomicBoolean isFailureThresholdBreached = new AtomicBoolean(false);
 
-        String collectionLink = getCollectionLink(request);
-
         this.partitionKeyRangeToLocationSpecificUnavailabilityInfo.compute(partitionKeyRangeWrapper, (partitionKeyRangeWrapperAsKey, partitionLevelLocationUnavailabilityInfoAsVal) -> {
 
             if (partitionLevelLocationUnavailabilityInfoAsVal == null) {
@@ -199,9 +200,9 @@ public List<String> getUnavailableRegionsForPartitionKeyRange(String collectionR
     private Flux<?> updateStaleLocationInfo() {
         return Mono.just(1)
             .delayElement(Duration.ofSeconds(Configs.getStalePartitionUnavailabilityRefreshIntervalInSeconds()))
-            .repeat()
+            .repeat(() -> !this.isClosed.get())
             .flatMap(ignore -> Flux.fromIterable(this.partitionKeyRangesWithPossibleUnavailableRegions.entrySet()))
-            .publishOn(CosmosSchedulers.PARTITION_AVAILABILITY_STALENESS_CHECK_SINGLE)
+            .publishOn(this.partitionRecoveryScheduler)
             .flatMap(partitionKeyRangeWrapperToPartitionKeyRangeWrapperPair -> {
 
                 logger.debug("Background updateStaleLocationInfo kicking in...");
@@ -258,7 +259,7 @@ private Flux<?> updateStaleLocationInfo() {
 
                             return gatewayAddressCache
                                 .submitOpenConnectionTasks(partitionKeyRangeWrapper.getPartitionKeyRange(), partitionKeyRangeWrapper.getCollectionResourceId())
-                                .publishOn(CosmosSchedulers.PARTITION_AVAILABILITY_STALENESS_CHECK_SINGLE)
+                                .publishOn(this.partitionRecoveryScheduler)
                                 .timeout(Duration.ofSeconds(Configs.getConnectionEstablishmentTimeoutForPartitionRecoveryInSeconds()))
                                 .doOnComplete(() -> {
 
@@ -348,6 +349,12 @@ public void setGlobalAddressResolver(GlobalAddressResolver globalAddressResolver
         this.globalAddressResolverSnapshot.set(globalAddressResolver);
     }
 
+    @Override
+    public void close() {
+        this.isClosed.set(true);
+        this.partitionRecoveryScheduler.dispose();
+    }
+
     private class PartitionLevelLocationUnavailabilityInfo {
 
         private final ConcurrentHashMap<URI, LocationSpecificHealthContext> locationEndpointToLocationSpecificContextForPartition;

diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json
@@ -98,13 +98,14 @@
       "DESIRED_CONSISTENCIES": "[\"Session\"]",
       "ACCOUNT_CONSISTENCY": "Session",
       "ArmConfig": {
-        "MultiMaster_MultiRegion": {
+        "MultiMaster_MultiRegion_CircuitBreaker_True": {
           "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }",
           "PREFERRED_LOCATIONS": "[\"East US 2\"]"
         }
       },
       "PROTOCOLS": "[\"Tcp\"]",
-      "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pmulti-master", "-Pflaky-multi-master", "-Pcircuit-breaker-misc-direct", "-Pcircuit-breaker-misc-gateway", "-Pcircuit-breaker-read-all-read-many", "-Pfast", "-Pdirect" ],
+      "ProfileFlag": [ "-Pmulti-master" ],
+      "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"",
       "Agent": {
         "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
       }
@@ -113,13 +114,29 @@
       "DESIRED_CONSISTENCIES": "[\"Session\"]",
       "ACCOUNT_CONSISTENCY": "Session",
       "ArmConfig": {
-        "SingleMaster_MultiRegion": {
-          "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true; enablePartitionMerge = $true}"
+        "MultiMaster_MultiRegion_CircuitBreaker_False": {
+          "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }",
+          "PREFERRED_LOCATIONS": "[\"East US 2\"]"
+        }
+      },
+      "PROTOCOLS": "[\"Tcp\"]",
+      "ProfileFlag": [ "-Pmulti-master" ],
+      "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"",
+      "Agent": {
+        "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
+      }
+    },
+    {
+      "DESIRED_CONSISTENCIES": "[\"Session\"]",
+      "ACCOUNT_CONSISTENCY": "Session",
+      "ArmConfig": {
+        "MultiMaster_MultiRegion": {
+          "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }",
+          "PREFERRED_LOCATIONS": "[\"East US 2\"]"
         }
       },
-      "ProfileFlag": "-Pmulti-region",
       "PROTOCOLS": "[\"Tcp\"]",
-      "PREFERRED_LOCATIONS": null,
+      "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pflaky-multi-master", "-Pcircuit-breaker-misc-direct", "-Pcircuit-breaker-misc-gateway", "-Pcircuit-breaker-read-all-read-many", "-Pfast", "-Pdirect" ],
       "Agent": {
         "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
       }