AER-2539 error handling in service (#45)

* Add retry functionality in BingSearchService
There's a max requests per second Bing supports with non-enterprise keys (and probably also with enterprise keys, but it should be higher in that case).
If we hit that, wait a second before trying again. 1 second is pretty long, but as we're using the async process anyway, I think it's OK to do this(?)
* Better error handling in AsyncSearchTaskDelegator
As it was, a NullpointerException in a search service would cause the delegator to also throw an exception, and a task would never be set to completed. This would cause the client to wait forever (well, wait, it would keep on sending a where-are-my-results-request every 250ms) and the user would never get an indication that something failed.
The `doOnError` method does not keep the error from flowing down, it just does its action and then the subscriber will still have to handle the original exception. And as we didn't define an error handler there, another exception would pop up.
Instead of silently ignoring the error and just returning results from all other services, which would also be an option, I went with returning a pretty generic 'Failure during search' result that a user will receive. This hopefully should trigger the user to warn us about it, so we can investigate what is going wrong. For good measure did the same for the synchronous version.

search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingSearchService.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
@@ -76,6 +77,7 @@ public class BingSearchService implements SearchTaskService {
  * https://docs.microsoft.com/en-us/bingmaps/getting-started/bing-maps-api-best-practices#reducing-usage-transactions
  */
  @Value("${nl.aerius.bing.apiKey:#{null}}") private String apiKey;
+ @Value("${nl.aerius.bing.maxRetries:3}") private int maxRetries;
 
  @Override
  public Single<SearchTaskResult> retrieveSearchResults(final String query) {
@@ -119,9 +121,26 @@ private void retrieveSuggestions(final String query, final Map<String, SearchSug
  }
 
  private JSONArray obtainResources(final String url) {
- final HttpResponse<JsonNode> json = Unirest.get(url).asJson();
- final JSONObject body = json.getBody().getObject();
- return body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources");
+ JSONObject body = null;
+ int retry = 0;
+ while (retry++ < maxRetries) {
+ final HttpResponse<JsonNode> json = Unirest.get(url).asJson();
+ body = json.getBody().getObject();
+ final int statusCode = body.getInt("statusCode");
+ if (statusCode == 200) {
+ return body.getJSONArray("resourceSets").getJSONObject(0).getJSONArray("resources");
+ } else if (statusCode == 429) {
+ LOG.info("Got too many retries status code from Bing, attempt {}.", retry);
+ try {
+ TimeUnit.SECONDS.sleep(1);
+ } catch (final InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ } else {
+ throw new BingServiceException("Unexpected status code: " + statusCode);
+ }
+ }
+ throw new BingServiceException("Retries failed, last returned: " + body);
  }
 
  private SuggestedLocation createSuggestedLocation(final JSONObject jsonObject) {

search-service-bing-geocoder/src/main/java/nl/aerius/search/tasks/BingServiceException.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright the State of the Netherlands
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/.
+ */
+package nl.aerius.search.tasks;
+
+public class BingServiceException extends RuntimeException {
+
+ public BingServiceException(final String message) {
+ super(message);
+ }
+}

search-service-bing-geocoder/src/test/java/nl/aerius/search/tasks/BingSearchServiceTest.java

@@ -45,5 +45,11 @@ void testWorksAtAll() {
  final SearchTaskResult suggestions = result.blockingGet();
 
  assertEquals(5, suggestions.getSuggestions().size(), "Expected number of results for 'edin' (should include 'edinburgh')");
+
+ final Single<SearchTaskResult> resultYork = delegator.retrieveSearchResults("york");
+
+ final SearchTaskResult suggestionsYork = resultYork.blockingGet();
+
+ assertEquals(4, suggestionsYork.getSuggestions().size(), "Expected number of results for 'edin' (should include 'edinburgh')");
  }
 }

search-service-mocks/src/main/java/nl/aerius/search/tasks/MockRuntimeExceptionTask.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright the State of the Netherlands
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/.
+ */
+package nl.aerius.search.tasks;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import io.reactivex.rxjava3.core.Single;
+
+import nl.aerius.search.domain.SearchCapability;
+import nl.aerius.search.domain.SearchTaskResult;
+
+@Component
+@ImplementsCapability(SearchCapability.MOCK_RUNTIME_EXCEPTION)
+public class MockRuntimeExceptionTask implements SearchTaskService {
+ private static final Logger LOG = LoggerFactory.getLogger(MockRuntimeExceptionTask.class);
+
+ @Override
+ public Single<SearchTaskResult> retrieveSearchResults(final String query) {
+ LOG.debug("Mocking runtim exception for query [{}]", query);
+ throw new NullPointerException("Some mocked nullpointer");
+ }
+}

search-service/src/main/java/nl/aerius/search/tasks/async/AsyncSearchTaskDelegator.java

@@ -85,12 +85,14 @@ public SearchResult retrieveSearchResultsAsync(final String query, final Set<Cap
  .parallel()
  .runOn(Schedulers.io())
  .map(v -> v.retrieveSearchResults(query))
- .doOnError(e -> LOG.error("Error while executing search task:", e))
  .flatMap(Single::toFlowable)
- .doAfterNext(r -> task.complete(r))
+ .doAfterNext(task::complete)
  .sequential()
- .doOnComplete(() -> task.complete())
- .subscribe();
+ .doOnComplete(task::fullyComplete)
+ .subscribe(x -> {}, e -> {
+ LOG.error("General error while executing search task:", e);
+ task.failureComplete();
+ });
 
  disposables.put(task.getUuid(), disposable);
 

search-service/src/main/java/nl/aerius/search/tasks/async/SearchResult.java

@@ -25,6 +25,7 @@
 import org.slf4j.LoggerFactory;
 
 import nl.aerius.search.domain.SearchSuggestion;
+import nl.aerius.search.domain.SearchSuggestionBuilder;
 import nl.aerius.search.domain.SearchTaskResult;
 
 public class SearchResult {
@@ -45,7 +46,16 @@ public boolean isComplete() {
  return complete;
  }
 
- public void complete() {
+ public void failureComplete() {
+ if (LOG.isTraceEnabled()) {
+ LOG.error("Search task {} has completed with a failure.", uuid);
+ }
+ results.add(SearchSuggestionBuilder.create("Failure during search, please contact the helpdesk"));
+
+ complete = true;
+ }
+
+ public void fullyComplete() {
  if (LOG.isTraceEnabled()) {
  LOG.trace("Search task {} has fully completed.", uuid);
  }

search-service/src/main/java/nl/aerius/search/tasks/sync/BlockingSearchTaskDelegator.java

@@ -30,6 +30,7 @@
 import io.reactivex.rxjava3.schedulers.Schedulers;
 
 import nl.aerius.search.domain.SearchSuggestion;
+import nl.aerius.search.domain.SearchSuggestionBuilder;
 import nl.aerius.search.tasks.CapabilityKey;
 import nl.aerius.search.tasks.SearchTaskService;
 import nl.aerius.search.tasks.TaskFactory;
@@ -69,6 +70,10 @@ public List<SearchSuggestion> retrieveSearchResults(final String query, final Se
  .flatMap(v -> Flowable.fromIterable(v.getSuggestions()))
  .sorted(TaskUtils.getResultComparator())
  .toList()
+ .onErrorReturn(e -> {
+ LOG.error("General error while executing search task:", e);
+ return List.of(SearchSuggestionBuilder.create("Failure during search, please contact the helpdesk"));
+ })
  .blockingGet();
  }
 }

search-service/src/main/resources/templates/synchronous-form.html

@@ -47,6 +47,7 @@ <h5>Mocks</h5>
  <div><input type="checkbox" id="MOCK_5" data-name="MOCK_5" onchange="toggleCapability(this)"><label for="MOCK_5">mock_5 (5 sec delayed mock result)</label></div>
  <div><input type="checkbox" id="MOCK_GROUP_0" data-name="MOCK_GROUP_0" onchange="toggleCapability(this)"><label for="MOCK_GROUP_0">mock_group_0 (0 sec delayed mock group result)</label></div>
  <div><input type="checkbox" id="MOCK_GROUP_1" data-name="MOCK_GROUP_1" onchange="toggleCapability(this)"><label for="MOCK_GROUP_1">mock_group_1 (1 sec delayed mock group result)</label></div>
+ <div><input type="checkbox" id="MOCK_RUNTIME_EXCEPTION" data-name="MOCK_RUNTIME_EXCEPTION" onchange="toggleCapability(this)"><label for="MOCK_RUNTIME_EXCEPTION">mock_runtime_exception (instant runtime exception during search)</label></div>
  </div>
 
  <input class="submit" type="submit" text="Submit">

search-service/src/test/java/nl/aerius/search/tasks/async/AsyncSearchTaskDelegatorTest.java

@@ -33,6 +33,7 @@
 import nl.aerius.search.tasks.CapabilityKey;
 import nl.aerius.search.tasks.Mock0SecondTask;
 import nl.aerius.search.tasks.MockHalfSecondTask;
+import nl.aerius.search.tasks.MockRuntimeExceptionTask;
 import nl.aerius.search.tasks.MockTenthSecondTask;
 import nl.aerius.search.tasks.SearchTaskService;
 import nl.aerius.search.tasks.TaskFactory;
@@ -69,14 +70,17 @@ void testResponseDelays() throws InterruptedException {
  TimeUnit.MILLISECONDS.sleep(50);
  final SearchResult result3 = delegator.retrieveSearchTask(uuid);
  assertEquals(1, result3.getResults().size(), "Should have 1 results at this point.");
+ assertFalse(result3.isComplete(), "Result should not be complete at this point.");
 
  TimeUnit.MILLISECONDS.sleep(100);
  final SearchResult result4 = delegator.retrieveSearchTask(uuid);
  assertEquals(2, result4.getResults().size(), "Should have 2 results at this point.");
+ assertFalse(result4.isComplete(), "Result should not be complete at this point.");
 
  TimeUnit.MILLISECONDS.sleep(400);
  final SearchResult result5 = delegator.retrieveSearchTask(uuid);
  assertEquals(3, result5.getResults().size(), "Should have 3 results at this point.");
+ assertTrue(result5.isComplete(), "Result should now be complete at this point.");
  }
 
  @Test
@@ -98,4 +102,35 @@ void testResponseCancellation() throws InterruptedException {
  assertTrue(result1.getResults().isEmpty(),
  "Results should still be empty after the task completes (>100ms) - the individual task was not cancelled");
  }
+
+ @Test
+ void testResponseRuntimeException() throws InterruptedException {
+ final SearchTaskService mock0Task = new Mock0SecondTask();
+ final SearchTaskService mock01Task = new MockTenthSecondTask();
+ final SearchTaskService mockRuntimeException = new MockRuntimeExceptionTask();
+
+ final Set<SearchTaskService> tasks = Set.of(mock0Task, mock01Task, mockRuntimeException);
+
+ final TaskFactory factory = new TaskFactory(tasks);
+ factory.onFactoryConstructed();
+ delegator = new AsyncSearchTaskDelegator(factory);
+
+ final Set<CapabilityKey> caps = Set.of(CapabilityKey.of(SearchCapability.MOCK_0, SearchRegion.NL),
+ CapabilityKey.of(SearchCapability.MOCK_01, SearchRegion.NL),
+ CapabilityKey.of(SearchCapability.MOCK_RUNTIME_EXCEPTION, SearchRegion.NL));
+ final SearchResult result1 = delegator.retrieveSearchResultsAsync("test", caps);
+ final String uuid = result1.getUuid();
+
+ final SearchResult result2 = delegator.retrieveSearchTask(uuid);
+
+ assertFalse(result2.isComplete(), "Result should not be complete at this point.");
+ assertNotNull(result2.getResults(), "Results should not be null at this point.");
+
+ TimeUnit.MILLISECONDS.sleep(50);
+ final SearchResult result3 = delegator.retrieveSearchTask(uuid);
+ assertEquals(1, result3.getResults().size(), "Should have 1 results at this point.");
+ assertTrue(result3.isComplete(), "Result should now be complete at this point, runtime exception should have triggered.");
+ assertEquals("Failure during search, please contact the helpdesk", result3.getResults().get(0).getDescription(),
+ "Description in case of failure");
+ }
 }

search-shared/src/main/java/nl/aerius/search/domain/SearchCapability.java

@@ -36,7 +36,9 @@ public enum SearchCapability {
  MOCK_5,
 
  MOCK_GROUP_0,
- MOCK_GROUP_1;
+ MOCK_GROUP_1,
+
+ MOCK_RUNTIME_EXCEPTION;
 
  public static SearchCapability safeValueOf(final String name) {
  try {