-
Notifications
You must be signed in to change notification settings - Fork 96
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
255 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
37 changes: 37 additions & 0 deletions
37
...e/src/main/java/com/yahoo/bard/webservice/application/healthchecks/LookupHealthCheck.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| // Copyright 2018 Yahoo Inc. | ||
| // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. | ||
| package com.yahoo.bard.webservice.application.healthchecks; | ||
|
|
||
| import com.yahoo.bard.webservice.metadata.LookupMetadataLoadTask; | ||
|
|
||
| import com.codahale.metrics.health.HealthCheck; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| import javax.inject.Singleton; | ||
|
|
||
| /** | ||
| * Check load statuses of all Druid lookups. | ||
| */ | ||
| @Singleton | ||
| public class LookupHealthCheck extends HealthCheck { | ||
| private final LookupMetadataLoadTask lookupMetadataLoadTask; | ||
|
|
||
| /** | ||
| * Constructor. | ||
| * | ||
| * @param lookupMetadataLoadTask A {@link LookupMetadataLoadTask} that keeps load statuses of | ||
| * all Druid lookups | ||
| */ | ||
| public LookupHealthCheck(LookupMetadataLoadTask lookupMetadataLoadTask) { | ||
| this.lookupMetadataLoadTask = lookupMetadataLoadTask; | ||
| } | ||
|
|
||
| @Override | ||
| public Result check() { | ||
| Set<String> unloadedLookups = lookupMetadataLoadTask.getPendingLookups(); | ||
| return unloadedLookups.isEmpty() | ||
| ? Result.healthy("All Druid lookups have been loaded.") | ||
| : Result.unhealthy("Lookups %s are not loaded.", unloadedLookups); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
124 changes: 124 additions & 0 deletions
124
fili-core/src/main/java/com/yahoo/bard/webservice/metadata/LookupMetadataLoadTask.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| // Copyright 2018 Yahoo Inc. | ||
| // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. | ||
| package com.yahoo.bard.webservice.metadata; | ||
|
|
||
| import com.yahoo.bard.webservice.application.LoadTask; | ||
| import com.yahoo.bard.webservice.config.SystemConfig; | ||
| import com.yahoo.bard.webservice.config.SystemConfigProvider; | ||
| import com.yahoo.bard.webservice.data.dimension.DimensionDictionary; | ||
| import com.yahoo.bard.webservice.data.dimension.impl.LookupDimension; | ||
| import com.yahoo.bard.webservice.druid.client.DruidWebService; | ||
| import com.yahoo.bard.webservice.druid.client.FailureCallback; | ||
| import com.yahoo.bard.webservice.druid.client.HttpErrorCallback; | ||
| import com.yahoo.bard.webservice.druid.client.SuccessCallback; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * Lookup Load task sends requests to Druid coordinator and returns list of configured lookup statuses in Druid. | ||
| */ | ||
| public class LookupMetadataLoadTask extends LoadTask<Boolean> { | ||
| private static final SystemConfig SYSTEM_CONFIG = SystemConfigProvider.getInstance(); | ||
|
|
||
| /** | ||
| * Location of lookup statuses on Druid coordinator. | ||
| */ | ||
| public static final String LOOKUP_QUERY = "/lookups/status/__default"; | ||
| /** | ||
| * Time between 2 consecutive lookup loading call in milliseconds. | ||
| */ | ||
| public static final String LOOKUP_NORMAL_CHECKING_PERIOD_KEY = SYSTEM_CONFIG.getPackageVariableName( | ||
| "lookup_normal_checking_period" | ||
| ); | ||
| /** | ||
| * Wait on https://github.com/yahoo/fili/issues/619. | ||
| */ | ||
| public static final String LOOKUP_ERROR_CHECKING_PERIOD_KEY = SYSTEM_CONFIG.getPackageVariableName( | ||
| "lookup_error_checking_period" | ||
| ); | ||
| /** | ||
| * Parameter specifying the delay before the first run of {@link LookupMetadataLoadTask}, in milliseconds. | ||
| */ | ||
| public static final String INITIAL_LOOKUP_CHECKING_DELAY = SYSTEM_CONFIG.getPackageVariableName( | ||
| "initial_lookup_checking_delay" | ||
| ); | ||
|
|
||
| private final DruidWebService druidClient; | ||
| private final DimensionDictionary dimensionDictionary; | ||
| private final SuccessCallback successCallback; | ||
| private final FailureCallback failureCallback; | ||
| private final HttpErrorCallback errorCallback; | ||
| private Set<String> pendingLookups; | ||
|
|
||
| /** | ||
| * Constructor. | ||
| * | ||
| * @param druidClient The client to query Druid coordinator | ||
| * @param dimensionDictionary A {@link com.yahoo.bard.webservice.data.dimension.DimensionDictionary} that is used | ||
| * to obtain a list of lookups in Fili. | ||
| */ | ||
| public LookupMetadataLoadTask(DruidWebService druidClient, DimensionDictionary dimensionDictionary) { | ||
| super( | ||
| LookupMetadataLoadTask.class.getSimpleName(), | ||
| SYSTEM_CONFIG.getLongProperty(INITIAL_LOOKUP_CHECKING_DELAY, 0), | ||
| SYSTEM_CONFIG.getLongProperty(LOOKUP_NORMAL_CHECKING_PERIOD_KEY, TimeUnit.MINUTES.toMillis(1)) | ||
| ); | ||
| this.druidClient = druidClient; | ||
| this.dimensionDictionary = dimensionDictionary; | ||
| this.successCallback = buildLookupSuccessCallback(); | ||
| this.failureCallback = getFailureCallback(); | ||
| this.errorCallback = getErrorCallback(); | ||
| } | ||
|
|
||
| @Override | ||
| public void run() { | ||
| // download load statuses of all lookups | ||
| druidClient.getJsonObject(successCallback, errorCallback, failureCallback, LOOKUP_QUERY); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a set of lookup namespaces that have not been loaded to Druid yet. | ||
| * | ||
| * @return the set of lookup namespaces that have not been loaded to Druid yet | ||
| */ | ||
| public Set<String> getPendingLookups() { | ||
| return pendingLookups; | ||
| } | ||
|
|
||
| /** | ||
| * Returns a callback that has actions on lookup metadata from a successful Druid response. | ||
| * <p> | ||
| * The callback obtains a complete list of configured lookups from Druid coordinator, compares this list against | ||
| * the list of lookups configured in Fili, and finds all lookup namespace names that are either not loaded yet in | ||
| * Druid or does not exist in Druid at all. These namespaces can be retrieved later by calling | ||
| * {@link #getPendingLookups()}. | ||
| * | ||
| * @return the callback that has actions on lookups from a successful Druid response | ||
| */ | ||
| protected SuccessCallback buildLookupSuccessCallback() { | ||
| return rootNode -> { | ||
| Map<String, Boolean> lookupStatuses = new HashMap<>(); | ||
| Iterator<Map.Entry<String, JsonNode>> entries = rootNode.fields(); | ||
| while (entries.hasNext()) { | ||
| Map.Entry<String, JsonNode> entry = entries.next(); | ||
| lookupStatuses.put(entry.getKey(), entry.getValue().get("loaded").asBoolean()); | ||
| } | ||
|
|
||
| pendingLookups = dimensionDictionary.findAll().stream() | ||
| .filter(dimension -> dimension instanceof LookupDimension) | ||
| .map(dimension -> (LookupDimension) dimension) | ||
| .map(LookupDimension::getNamespaces) | ||
| .flatMap(List::stream) | ||
| .filter(namespace -> !lookupStatuses.containsKey(namespace) || !lookupStatuses.get(namespace)) | ||
| .collect(Collectors.toSet()); | ||
| }; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
...core/src/test/groovy/com/yahoo/bard/webservice/metadata/LookupMetadataLoadTaskSpec.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // Copyright 2018 Yahoo Inc. | ||
| // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. | ||
| package com.yahoo.bard.webservice.metadata | ||
|
|
||
| import com.yahoo.bard.webservice.data.dimension.DimensionDictionary | ||
| import com.yahoo.bard.webservice.data.dimension.impl.LookupDimension | ||
| import com.yahoo.bard.webservice.druid.client.DruidWebService | ||
| import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| class LookupMetadataLoadTaskSpec extends Specification { | ||
| DruidWebService druidClient | ||
|
|
||
| LookupDimension lookupDimension | ||
| DimensionDictionary dimensionDictionary | ||
|
|
||
| LookupMetadataLoadTask lookupLoadTask | ||
|
|
||
| def setup() { | ||
| druidClient = new TestDruidWebService() | ||
| druidClient.jsonResponse = { | ||
| """ | ||
| { | ||
| "loadedLookup": { | ||
| "loaded": true | ||
| }, | ||
| "pendingLookup": { | ||
| "loaded": false | ||
| } | ||
| } | ||
| """ | ||
| } | ||
|
|
||
| lookupDimension = Mock(LookupDimension) | ||
| lookupDimension.getNamespaces() >> ["loadedLookup", "pendingLookup", "LookupNotInDruid"] | ||
|
|
||
| lookupLoadTask = new LookupMetadataLoadTask(druidClient, new DimensionDictionary([lookupDimension] as Set)) | ||
| } | ||
|
|
||
| def "LookupLoadTask, when runs, finds pending lookups"() { | ||
| when: | ||
| lookupLoadTask.run() | ||
|
|
||
| then: | ||
| lookupLoadTask.getPendingLookups() == ["pendingLookup", "LookupNotInDruid"] as Set | ||
| } | ||
| } |