-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fixes yet another place where we prefetch too aggressively in the que…
…ry path (#27299) * Fixing NPE in CosmosPagedFlux when trying to access non-existing diagnostics * Iterating on the change to limit prefetch memory consumption in CosmosPagedIterable in Spark query/change feed scenarios * Fixing tests * Fixing typo in comment Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com> * Updating changelogs * Fixes yet another place where we prefetch too aggressively in the query path * Update PaginatorTest.java * Updated changelogs * Fixing unintentional reformatting Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com> * Update Paginator.java * Update ItemsPartitionReader.scala Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com>
- Loading branch information
1 parent
c3e9b68
commit cc10d8a
Showing
7 changed files
with
103 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
...osmos/azure-cosmos/src/test/java/com/azure/cosmos/implementation/query/PaginatorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| package com.azure.cosmos.implementation.query; | ||
|
|
||
| import com.azure.cosmos.models.CosmosQueryRequestOptions; | ||
| import com.azure.cosmos.models.ModelBridgeInternal; | ||
| import org.testng.annotations.DataProvider; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| public class PaginatorTest { | ||
| @Test(groups = { "unit" }, dataProvider = "queryParams") | ||
| public void preFetchCalculation( | ||
| CosmosQueryRequestOptions testQueryOptions, | ||
| int testTop, | ||
| int testPageSize, | ||
| int expectedPreFetchCount) { | ||
|
|
||
| assertThat( | ||
| Paginator | ||
| .getPreFetchCount(testQueryOptions, testTop, testPageSize) | ||
| ).isEqualTo(expectedPreFetchCount); | ||
| } | ||
|
|
||
| @DataProvider(name = "queryParams") | ||
| public static Object[][] queryParamProvider() { | ||
|
|
||
| CosmosQueryRequestOptions optionsWithBufferSizeTenThousandAndDifferentMaxItemCount = | ||
| new CosmosQueryRequestOptions().setMaxBufferedItemCount(10_000); | ||
| // initial continuation token | ||
| ModelBridgeInternal | ||
| .setQueryRequestOptionsContinuationTokenAndMaxItemCount( | ||
| optionsWithBufferSizeTenThousandAndDifferentMaxItemCount, | ||
| "someContinuation", | ||
| 1); // maxItemCount 1 to test that explicit page Size trumps query options | ||
|
|
||
| CosmosQueryRequestOptions optionsWithBufferSizeTenThousand = | ||
| new CosmosQueryRequestOptions().setMaxBufferedItemCount(10_000); | ||
| // initial continuation token | ||
| ModelBridgeInternal | ||
| .setQueryRequestOptionsContinuationTokenAndMaxItemCount( | ||
| optionsWithBufferSizeTenThousand, | ||
| "someContinuation", | ||
| 1_000); | ||
|
|
||
| CosmosQueryRequestOptions optionsWithMaxIntAsMaxItemCount = | ||
| new CosmosQueryRequestOptions().setMaxBufferedItemCount(Integer.MAX_VALUE); | ||
|
|
||
| return new Object[][] { | ||
| //options, top, pageSize, expectedPreFetchCount | ||
| { optionsWithBufferSizeTenThousand, -1, 1_000, 10 }, // top ignored | ||
| { optionsWithBufferSizeTenThousandAndDifferentMaxItemCount, -1, 1_000, 10 }, // explicit pageSize wins | ||
| { optionsWithBufferSizeTenThousand, 0, 1_000, 10 }, // top ignored | ||
| { optionsWithBufferSizeTenThousand, 500, 1_000, 20 }, // effective page size is top | ||
| { optionsWithBufferSizeTenThousand, 100, 1_000, 32 }, // effective page size is top - should result in 100 | ||
| // but max prefetch count is 32 | ||
| { optionsWithBufferSizeTenThousand, -1, -1, 32 }, // effective pageSize is at least 1 | ||
| { optionsWithBufferSizeTenThousand, -1, 0, 32 }, // effective pageSize is at least 1 | ||
| { optionsWithBufferSizeTenThousand, -1, 20_000, 1 }, // at least 1 page is buffered even when | ||
| // maxBufferedItemCount < maxItemCount | ||
| { optionsWithMaxIntAsMaxItemCount, -1, Integer.MAX_VALUE, 1 }, // Exactly 1 page is buffered when | ||
| // maxBufferedItemCount == maxItemCount | ||
| }; | ||
| } | ||
| } |