feat(clients): add optionnal scopes to replaceAllObjects [skip-bc] (#4296)

Author: millotp

clients/algoliasearch-client-csharp/algoliasearch/Utils/SearchClientExtensions.cs

@@ -140,11 +140,12 @@ public partial interface ISearchClient
   /// <param name="indexName">The index in which to perform the request.</param>
   /// <param name="objects">The list of `objects` to store in the given Algolia `indexName`.</param>
   /// <param name="batchSize">The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.</param>
+  /// <param name="scopes"> The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].</param>
   /// <param name="options">Add extra http header or query parameters to Algolia.</param>
   /// <param name="cancellationToken">Cancellation Token to cancel the request.</param>
-  Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
+  Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
   /// <inheritdoc cref="ReplaceAllObjectsAsync{T}(string, IEnumerable{T}, int, RequestOptions, CancellationToken)"/>
-  ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
+  ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
 
   /// <summary>
   /// Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
@@ -484,21 +485,26 @@ private static int NextDelay(int retryCount)
 
   /// <inheritdoc/>
   public async Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects,
-    int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
+    int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
   {
     if (objects == null)
     {
       throw new ArgumentNullException(nameof(objects));
     }
 
+    if (scopes == null)
+    {
+      scopes = new List<ScopeType> { ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms };
+    }
+
     var rnd = new Random();
     var tmpIndexName = $"{indexName}_tmp_{rnd.Next(100)}";
 
     try
     {
       var copyResponse = await OperationIndexAsync(indexName,
           new OperationIndexParams(OperationType.Copy, tmpIndexName)
-          { Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
+          { Scope = scopes }, options, cancellationToken)
         .ConfigureAwait(false);
 
       var batchResponse = await ChunkedBatchAsync(tmpIndexName, objects, Action.AddObject, true, batchSize,
@@ -509,7 +515,7 @@ await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: option
 
       copyResponse = await OperationIndexAsync(indexName,
           new OperationIndexParams(OperationType.Copy, tmpIndexName)
-          { Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
+          { Scope = scopes }, options, cancellationToken)
         .ConfigureAwait(false);
       await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: options, ct: cancellationToken)
         .ConfigureAwait(false);
@@ -537,9 +543,9 @@ await WaitForTaskAsync(tmpIndexName, moveResponse.TaskID, requestOptions: option
   }
 
   /// <inheritdoc/>
-  public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000,
+  public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null,
     RequestOptions options = null, CancellationToken cancellationToken = default) where T : class =>
-    AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, options, cancellationToken));
+    AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, scopes, options, cancellationToken));
 
   /// <inheritdoc/>
   public async Task<List<BatchResponse>> ChunkedBatchAsync<T>(string indexName, IEnumerable<T> objects,

clients/algoliasearch-client-go/.golangci.yml

@@ -61,6 +61,7 @@ linters:
 
     # Deprecated
     - execinquery
+    - exportloopref
 
 issues:
   exclude-generated: disable

clients/algoliasearch-client-kotlin/client/src/commonMain/kotlin/com/algolia/client/extensions/SearchClient.kt

@@ -462,12 +462,14 @@ public suspend fun SearchClient.partialUpdateObjects(
  * @param indexName The index in which to perform the request.
  * @param objects The list of objects to replace.
  * @param batchSize The size of the batch. Default is 1000.
+ * @param scopes The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
  * @return responses from the three-step operations: copy, batch, move.
  */
 public suspend fun SearchClient.replaceAllObjects(
   indexName: String,
   objects: List<JsonObject>,
   batchSize: Int = 1000,
+  scopes: List<ScopeType> = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
   requestOptions: RequestOptions? = null,
 ): ReplaceAllObjectsResponse {
   val tmpIndexName = "${indexName}_tmp_${Random.nextInt(from = 0, until = 100)}"
@@ -478,7 +480,7 @@ public suspend fun SearchClient.replaceAllObjects(
       operationIndexParams = OperationIndexParams(
         operation = OperationType.Copy,
         destination = tmpIndexName,
-        scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
+        scope = scopes,
       ),
       requestOptions = requestOptions,
     )
@@ -499,7 +501,7 @@ public suspend fun SearchClient.replaceAllObjects(
       operationIndexParams = OperationIndexParams(
         operation = OperationType.Copy,
         destination = tmpIndexName,
-        scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
+        scope = scopes,
       ),
       requestOptions = requestOptions,
     )

clients/algoliasearch-client-scala/src/main/scala/algoliasearch/extension/package.scala

@@ -353,6 +353,8 @@ package object extension {
       *   The list of objects to replace.
       * @param batchSize
       *   The size of the batch. Default is 1000.
+      * @param scopes
+      *   The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
       * @param requestOptions
       *   Additional request configuration.
       * @return
@@ -362,6 +364,7 @@ package object extension {
         indexName: String,
         objects: Seq[Any],
         batchSize: Int = 1000,
+        scopes: Option[Seq[ScopeType]] = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms)),
         requestOptions: Option[RequestOptions] = None
     )(implicit ec: ExecutionContext): Future[ReplaceAllObjectsResponse] = {
       val tmpIndexName = s"${indexName}_tmp_${scala.util.Random.nextInt(100)}"
@@ -373,7 +376,7 @@ package object extension {
             operationIndexParams = OperationIndexParams(
               operation = OperationType.Copy,
               destination = tmpIndexName,
-              scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
+              scope = scopes
             ),
             requestOptions = requestOptions
           )
@@ -394,7 +397,7 @@ package object extension {
             operationIndexParams = OperationIndexParams(
               operation = OperationType.Copy,
               destination = tmpIndexName,
-              scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
+              scope = scopes
             ),
             requestOptions = requestOptions
           )

clients/algoliasearch-client-swift/Sources/Search/Extra/SearchClientExtension.swift

@@ -548,13 +548,15 @@ public extension SearchClient {
     /// - parameter indexName: The name of the index where to replace the objects
     /// - parameter objects: The new objects
     /// - parameter batchSize: The maximum number of objects to include in a batch
+    /// - parameter scopes: The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms']
     /// - parameter requestOptions: The request options
     /// - returns: ReplaceAllObjectsResponse
     @discardableResult
     func replaceAllObjects(
         indexName: String,
         objects: [some Encodable],
         batchSize: Int = 1000,
+        scopes: [ScopeType] = [.settings, .rules, .synonyms],
         requestOptions: RequestOptions? = nil
     ) async throws -> ReplaceAllObjectsResponse {
         let tmpIndexName = "\(indexName)_tmp_\(Int.random(in: 1_000_000 ..< 10_000_000))"
@@ -565,7 +567,7 @@ public extension SearchClient {
                 operationIndexParams: OperationIndexParams(
                     operation: .copy,
                     destination: tmpIndexName,
-                    scope: [.settings, .rules, .synonyms]
+                    scope: scopes
                 ),
                 requestOptions: requestOptions
             )
@@ -584,7 +586,7 @@ public extension SearchClient {
                 operationIndexParams: OperationIndexParams(
                     operation: .copy,
                     destination: tmpIndexName,
-                    scope: [.settings, .rules, .synonyms]
+                    scope: scopes
                 ),
                 requestOptions: requestOptions
             )

scripts/cts/runCts.ts

@@ -10,6 +10,7 @@ import { assertChunkWrapperValid } from './testServer/chunkWrapper.js';
 import { startTestServer } from './testServer/index.js';
 import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.js';
 import { assertValidReplaceAllObjectsFailed } from './testServer/replaceAllObjectsFailed.js';
+import { assertValidReplaceAllObjectsScopes } from './testServer/replaceAllObjectsScopes.js';
 import { assertValidTimeouts } from './testServer/timeout.js';
 import { assertValidWaitForApiKey } from './testServer/waitFor.js';
 
@@ -151,6 +152,7 @@ export async function runCts(
     assertChunkWrapperValid(languages.length - skip('dart') - skip('scala'));
     assertValidReplaceAllObjects(languages.length - skip('dart') - skip('scala'));
     assertValidReplaceAllObjectsFailed(languages.length - skip('dart') - skip('scala'));
+    assertValidReplaceAllObjectsScopes(languages.length - skip('dart') - skip('scala'));
     assertValidWaitForApiKey(languages.length - skip('dart') - skip('scala'));
   }
   if (withBenchmarkServer) {

scripts/cts/testServer/index.ts

@@ -13,6 +13,7 @@ import { chunkWrapperServer } from './chunkWrapper.js';
 import { gzipServer } from './gzip.js';
 import { replaceAllObjectsServer } from './replaceAllObjects.js';
 import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.js';
+import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.js';
 import { timeoutServer } from './timeout.js';
 import { timeoutServerBis } from './timeoutBis.js';
 import { waitForApiKeyServer } from './waitFor.js';
@@ -26,6 +27,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
       timeoutServerBis(),
       replaceAllObjectsServer(),
       replaceAllObjectsServerFailed(),
+      replaceAllObjectsScopesServer(),
       chunkWrapperServer(),
       waitForApiKeyServer(),
       apiKeyServer(),

scripts/cts/testServer/replaceAllObjectsScopes.ts

@@ -0,0 +1,122 @@
+import type { Server } from 'http';
+
+import { expect } from 'chai';
+import type { Express } from 'express';
+import express from 'express';
+
+import { setupServer } from './index.js';
+
+const raoState: Record<
+  string,
+  {
+    copyCount: number;
+    batchCount: number;
+    waitTaskCount: number;
+    tmpIndexName: string;
+    waitingForFinalWaitTask: boolean;
+    successful: boolean;
+  }
+> = {};
+
+export function assertValidReplaceAllObjectsScopes(expectedCount: number): void {
+  expect(Object.keys(raoState)).to.have.length(expectedCount);
+  for (const lang in raoState) {
+    expect(raoState[lang].successful).to.equal(true);
+  }
+}
+
+function addRoutes(app: Express): void {
+  app.use(express.urlencoded({ extended: true }));
+  app.use(
+    express.json({
+      type: ['application/json', 'text/plain'], // the js client sends the body as text/plain
+    }),
+  );
+
+  app.post('/1/indexes/:indexName/operation', (req, res) => {
+    expect(req.params.indexName).to.match(/^cts_e2e_replace_all_objects_scopes_(.*)$/);
+
+    switch (req.body.operation) {
+      case 'copy': {
+        expect(req.params.indexName).to.not.include('tmp');
+        expect(req.body.destination).to.include('tmp');
+        expect(req.body.scope).to.deep.equal(['settings', 'synonyms']);
+
+        const lang = req.params.indexName.replace('cts_e2e_replace_all_objects_scopes_', '');
+        if (!raoState[lang] || raoState[lang].successful) {
+          raoState[lang] = {
+            copyCount: 1,
+            batchCount: 0,
+            waitTaskCount: 0,
+            tmpIndexName: req.body.destination,
+            waitingForFinalWaitTask: false,
+            successful: false,
+          };
+        } else {
+          raoState[lang].copyCount++;
+        }
+
+        res.json({ taskID: 123 + raoState[lang].copyCount, updatedAt: '2021-01-01T00:00:00.000Z' });
+        break;
+      }
+      case 'move': {
+        const lang = req.body.destination.replace('cts_e2e_replace_all_objects_scopes_', '');
+        expect(raoState).to.include.keys(lang);
+        expect(raoState[lang]).to.deep.equal({
+          copyCount: 2,
+          batchCount: 2,
+          waitTaskCount: 3,
+          tmpIndexName: req.params.indexName,
+          waitingForFinalWaitTask: false,
+          successful: false,
+        });
+
+        expect(req.body.scope).to.equal(undefined);
+
+        raoState[lang].waitingForFinalWaitTask = true;
+
+        res.json({ taskID: 777, updatedAt: '2021-01-01T00:00:00.000Z' });
+
+        break;
+      }
+      default:
+        res.status(400).json({
+          message: `invalid operation: ${req.body.operation}, body: ${JSON.stringify(req.body)}`,
+        });
+    }
+  });
+
+  app.post('/1/indexes/:indexName/batch', (req, res) => {
+    const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
+    expect(raoState).to.include.keys(lang);
+    expect(req.body.requests.every((r) => r.action === 'addObject')).to.equal(true);
+
+    raoState[lang].batchCount += req.body.requests.length;
+
+    res.json({
+      taskID: 124 + raoState[lang].batchCount,
+      objectIDs: req.body.requests.map((r) => r.body.objectID),
+    });
+  });
+
+  app.get('/1/indexes/:indexName/task/:taskID', (req, res) => {
+    const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
+    expect(raoState).to.include.keys(lang);
+
+    raoState[lang].waitTaskCount++;
+    if (raoState[lang].waitingForFinalWaitTask) {
+      expect(req.params.taskID).to.equal('777');
+      expect(raoState[lang].waitTaskCount).to.equal(4);
+
+      raoState[lang].successful = true;
+    }
+
+    res.json({ status: 'published', updatedAt: '2021-01-01T00:00:00.000Z' });
+  });
+}
+
+export function replaceAllObjectsScopesServer(): Promise<Server> {
+  // this server is used to simulate the responses for the replaceAllObjects method with partial scopes,
+  // and uses a state machine to determine if the logic is correct.
+  return setupServer('replaceAllObjectsScopes', 6685, addRoutes);
+}

specs/search/helpers/replaceAllObjects.yml

@@ -40,6 +40,14 @@ method:
         required: false
         schema:
           type: integer
+      - in: query
+        name: scopes
+        description: List of scopes to kepp in the index. Defaults to `settings`, `synonyms`, and `rules`.
+        required: false
+        schema:
+          type: array
+          items:
+            $ref: '../common/enums.yml#/scopeType'
     responses:
       '200':
         description: OK