Skip to content

Commit

Permalink
feat(clients): cleanup after replaceAllObjects failure [skip-bc] (#3824
Browse files Browse the repository at this point in the history
…) (generated) [skip ci]

Co-authored-by: Pierre Millot <pierre.millot@algolia.com>
Co-authored-by: Thomas Raffray <Fluf22@users.noreply.github.com>
  • Loading branch information
3 people committed Dec 31, 2024
1 parent 37223c9 commit 02619a3
Show file tree
Hide file tree
Showing 38 changed files with 979 additions and 297 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
12 changes: 12 additions & 0 deletions clients/algoliasearch-client-go/algolia/search/api_search.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6572,24 +6572,6 @@ public <T> List<BatchResponse> chunkedBatch(
return chunkedBatch(indexName, objects, action, waitForTasks, 1000, requestOptions);
}

/**
* Push a new set of objects and remove all previous ones. Settings, synonyms and query rules are
* untouched. Replace all records in an index without any downtime. See
* https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation
* details.
*
* @param indexName The `indexName` to replace `objects` in.
* @param objects The array of `objects` to store in the given Algolia `indexName`.
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
* to `length(objects) / batchSize`.
* @throws AlgoliaRetryException When the retry has failed on all hosts
* @throws AlgoliaApiException When the API sends an http error code
* @throws AlgoliaRuntimeException When an error occurred during the serialization
*/
public <T> ReplaceAllObjectsResponse replaceAllObjects(String indexName, Iterable<T> objects, int batchSize) {
return replaceAllObjects(indexName, objects, batchSize, null);
}

/**
* Helper: Saves the given array of objects in the given index. The `chunkedBatch` helper is used
* under the hood, which creates a `batch` requests with at most 1000 objects in it.
Expand Down Expand Up @@ -6822,6 +6804,40 @@ public <T> List<BatchResponse> partialUpdateObjects(
);
}

/**
* Push a new set of objects and remove all previous ones. Settings, synonyms and query rules are
* untouched. Replace all records in an index without any downtime. See
* https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation
* details.
*
* @param indexName The `indexName` to replace `objects` in.
* @param objects The array of `objects` to store in the given Algolia `indexName`.
* @throws AlgoliaRetryException When the retry has failed on all hosts
* @throws AlgoliaApiException When the API sends an http error code
* @throws AlgoliaRuntimeException When an error occurred during the serialization
*/
public <T> ReplaceAllObjectsResponse replaceAllObjects(String indexName, Iterable<T> objects) {
return replaceAllObjects(indexName, objects, -1);
}

/**
* Push a new set of objects and remove all previous ones. Settings, synonyms and query rules are
* untouched. Replace all records in an index without any downtime. See
* https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation
* details.
*
* @param indexName The `indexName` to replace `objects` in.
* @param objects The array of `objects` to store in the given Algolia `indexName`.
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
* to `length(objects) / batchSize`.
* @throws AlgoliaRetryException When the retry has failed on all hosts
* @throws AlgoliaApiException When the API sends an http error code
* @throws AlgoliaRuntimeException When an error occurred during the serialization
*/
public <T> ReplaceAllObjectsResponse replaceAllObjects(String indexName, Iterable<T> objects, int batchSize) {
return replaceAllObjects(indexName, objects, batchSize, null);
}

/**
* Push a new set of objects and remove all previous ones. Settings, synonyms and query rules are
* untouched. Replace all records in an index without any downtime. See
Expand All @@ -6847,47 +6863,57 @@ public <T> ReplaceAllObjectsResponse replaceAllObjects(
Random rnd = new Random();
String tmpIndexName = indexName + "_tmp_" + rnd.nextInt(100);

// Copy settings, synonyms and rules
UpdatedAtResponse copyOperationResponse = operationIndex(
indexName,
new OperationIndexParams()
.setOperation(OperationType.COPY)
.setDestination(tmpIndexName)
.addScope(ScopeType.SETTINGS)
.addScope(ScopeType.RULES)
.addScope(ScopeType.SYNONYMS),
requestOptions
);
if (batchSize == -1) {
batchSize = 1000;
}

// Save new objects
List<BatchResponse> batchResponses = chunkedBatch(tmpIndexName, objects, Action.ADD_OBJECT, true, batchSize, requestOptions);
try {
// Copy settings, synonyms and rules
UpdatedAtResponse copyOperationResponse = operationIndex(
indexName,
new OperationIndexParams()
.setOperation(OperationType.COPY)
.setDestination(tmpIndexName)
.addScope(ScopeType.SETTINGS)
.addScope(ScopeType.RULES)
.addScope(ScopeType.SYNONYMS),
requestOptions
);

waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);
// Save new objects
List<BatchResponse> batchResponses = chunkedBatch(tmpIndexName, objects, Action.ADD_OBJECT, true, batchSize, requestOptions);

copyOperationResponse = operationIndex(
indexName,
new OperationIndexParams()
.setOperation(OperationType.COPY)
.setDestination(tmpIndexName)
.addScope(ScopeType.SETTINGS)
.addScope(ScopeType.RULES)
.addScope(ScopeType.SYNONYMS),
requestOptions
);
waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);
waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);

// Move temporary index to source index
UpdatedAtResponse moveOperationResponse = operationIndex(
tmpIndexName,
new OperationIndexParams().setOperation(OperationType.MOVE).setDestination(indexName),
requestOptions
);
waitForTask(tmpIndexName, moveOperationResponse.getTaskID(), requestOptions);
copyOperationResponse = operationIndex(
indexName,
new OperationIndexParams()
.setOperation(OperationType.COPY)
.setDestination(tmpIndexName)
.addScope(ScopeType.SETTINGS)
.addScope(ScopeType.RULES)
.addScope(ScopeType.SYNONYMS),
requestOptions
);
waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);

// Move temporary index to source index
UpdatedAtResponse moveOperationResponse = operationIndex(
tmpIndexName,
new OperationIndexParams().setOperation(OperationType.MOVE).setDestination(indexName),
requestOptions
);
waitForTask(tmpIndexName, moveOperationResponse.getTaskID(), requestOptions);

return new ReplaceAllObjectsResponse()
.setCopyOperationResponse(copyOperationResponse)
.setBatchResponses(batchResponses)
.setMoveOperationResponse(moveOperationResponse);
} catch (Exception e) {
deleteIndex(tmpIndexName);

return new ReplaceAllObjectsResponse()
.setCopyOperationResponse(copyOperationResponse)
.setBatchResponses(batchResponses)
.setMoveOperationResponse(moveOperationResponse);
throw e;
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -642,57 +642,63 @@ export function createSearchClient({
const randomSuffix = Math.floor(Math.random() * 1000000) + 100000;
const tmpIndexName = `${indexName}_tmp_${randomSuffix}`;

let copyOperationResponse = await this.operationIndex(
{
indexName,
operationIndexParams: {
operation: 'copy',
destination: tmpIndexName,
scope: ['settings', 'rules', 'synonyms'],
try {
let copyOperationResponse = await this.operationIndex(
{
indexName,
operationIndexParams: {
operation: 'copy',
destination: tmpIndexName,
scope: ['settings', 'rules', 'synonyms'],
},
},
},
requestOptions,
);
requestOptions,
);

const batchResponses = await this.chunkedBatch(
{ indexName: tmpIndexName, objects, waitForTasks: true, batchSize },
requestOptions,
);
const batchResponses = await this.chunkedBatch(
{ indexName: tmpIndexName, objects, waitForTasks: true, batchSize },
requestOptions,
);

await this.waitForTask({
indexName: tmpIndexName,
taskID: copyOperationResponse.taskID,
});
await this.waitForTask({
indexName: tmpIndexName,
taskID: copyOperationResponse.taskID,
});

copyOperationResponse = await this.operationIndex(
{
indexName,
operationIndexParams: {
operation: 'copy',
destination: tmpIndexName,
scope: ['settings', 'rules', 'synonyms'],
copyOperationResponse = await this.operationIndex(
{
indexName,
operationIndexParams: {
operation: 'copy',
destination: tmpIndexName,
scope: ['settings', 'rules', 'synonyms'],
},
},
},
requestOptions,
);
await this.waitForTask({
indexName: tmpIndexName,
taskID: copyOperationResponse.taskID,
});
requestOptions,
);
await this.waitForTask({
indexName: tmpIndexName,
taskID: copyOperationResponse.taskID,
});

const moveOperationResponse = await this.operationIndex(
{
const moveOperationResponse = await this.operationIndex(
{
indexName: tmpIndexName,
operationIndexParams: { operation: 'move', destination: indexName },
},
requestOptions,
);
await this.waitForTask({
indexName: tmpIndexName,
operationIndexParams: { operation: 'move', destination: indexName },
},
requestOptions,
);
await this.waitForTask({
indexName: tmpIndexName,
taskID: moveOperationResponse.taskID,
});
taskID: moveOperationResponse.taskID,
});

return { copyOperationResponse, batchResponses, moveOperationResponse };
return { copyOperationResponse, batchResponses, moveOperationResponse };
} catch (error) {
await this.deleteIndex({ indexName: tmpIndexName });

throw error;
}
},

async indexExists({ indexName }: GetSettingsProps): Promise<boolean> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ body:
id: client
attributes:
label: Client
description: Which API are you targetting?
description: Which API are you targeting?
options:
- All
- AB testing
Expand Down
Loading

0 comments on commit 02619a3

Please sign in to comment.