diff --git a/gradle.properties b/gradle.properties index 2e15781bf..fe820f826 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=1.2.19-SNAPSHOT +version=1.3.0-SNAPSHOT diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 9829a99a5..e230e2b1c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=http\://services.gradle.org/distributions/gradle-1.1-bin.zip +distributionUrl=http\://services.gradle.org/distributions/gradle-1.3-bin.zip diff --git a/hystrix-core/build.gradle b/hystrix-core/build.gradle index 31e6e939f..ccd65597c 100644 --- a/hystrix-core/build.gradle +++ b/hystrix-core/build.gradle @@ -4,6 +4,7 @@ apply plugin: 'idea' dependencies { compile 'com.netflix.archaius:archaius-core:0.4.1' + compile 'com.netflix.rxjava:rxjava-core:0.9.0' compile 'org.slf4j:slf4j-api:1.7.0' compile 'com.google.code.findbugs:jsr305:2.0.0' provided 'junit:junit-dep:4.10' diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java index aea37337e..fa7d15717 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java @@ -23,21 +23,13 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.concurrent.NotThreadSafe; -import javax.annotation.concurrent.ThreadSafe; import org.junit.After; import org.junit.Before; @@ -45,18 +37,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import rx.Observable; +import rx.Scheduler; +import rx.concurrency.Schedulers; +import rx.subjects.ReplaySubject; + import com.netflix.hystrix.HystrixCommand.UnitTest.TestHystrixCommand; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.collapser.CollapserTimer; +import com.netflix.hystrix.collapser.HystrixCollapserBridge; +import com.netflix.hystrix.collapser.RealCollapserTimer; +import com.netflix.hystrix.collapser.RequestCollapser; +import com.netflix.hystrix.collapser.RequestCollapserFactory; import com.netflix.hystrix.exception.HystrixRuntimeException; import com.netflix.hystrix.strategy.HystrixPlugins; -import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; -import com.netflix.hystrix.strategy.concurrency.HystrixContextCallable; import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableHolder; -import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle; -import com.netflix.hystrix.strategy.properties.HystrixPropertiesFactory; import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; -import com.netflix.hystrix.util.HystrixTimer; import com.netflix.hystrix.util.HystrixTimer.TimerListener; /** @@ -80,7 +78,11 @@ */ public abstract class HystrixCollapser implements HystrixExecutable { - private static final Logger logger = LoggerFactory.getLogger(HystrixCollapser.class); + static final Logger logger = LoggerFactory.getLogger(HystrixCollapser.class); + + private final RequestCollapserFactory collapserFactory; + private final HystrixRequestCache requestCache; + private final HystrixCollapserBridge collapserInstanceWrapper; /** * The scope of request collapsing. @@ -96,17 +98,6 @@ public static enum Scope { REQUEST, GLOBAL } - private final CollapserTimer timer; - private final HystrixCollapserKey collapserKey; - private final HystrixCollapserProperties properties; - private final Scope scope; - private final HystrixConcurrencyStrategy concurrencyStrategy; - - /* - * Instance of RequestCache logic - */ - private final HystrixRequestCache requestCache; - /** * Collapser with default {@link HystrixCollapserKey} derived from the implementing class name and scoped to {@link Scope#REQUEST} and default configuration. */ @@ -138,19 +129,51 @@ protected HystrixCollapser(Setter setter) { } private HystrixCollapser(HystrixCollapserKey collapserKey, Scope scope, CollapserTimer timer, HystrixCollapserProperties.Setter propertiesBuilder) { - /* strategy: ConcurrencyStrategy */ - this.concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); - - this.timer = timer; - this.scope = scope; if (collapserKey == null || collapserKey.name().trim().equals("")) { String defaultKeyName = getDefaultNameFromClass(getClass()); - this.collapserKey = HystrixCollapserKey.Factory.asKey(defaultKeyName); - } else { - this.collapserKey = collapserKey; + collapserKey = HystrixCollapserKey.Factory.asKey(defaultKeyName); } - this.requestCache = HystrixRequestCache.getInstance(this.collapserKey, this.concurrencyStrategy); - this.properties = HystrixPropertiesFactory.getCollapserProperties(this.collapserKey, propertiesBuilder); + + this.collapserFactory = new RequestCollapserFactory(collapserKey, scope, timer, propertiesBuilder); + this.requestCache = HystrixRequestCache.getInstance(collapserKey, HystrixPlugins.getInstance().getConcurrencyStrategy()); + + final HystrixCollapser self = this; + + /** + * Used to pass public method invocation to the underlying implementation in a separate package while leaving the methods 'protected' in this class. + */ + collapserInstanceWrapper = new HystrixCollapserBridge() { + + @Override + public Collection>> shardRequests(Collection> requests) { + return self.shardRequests(requests); + } + + @Override + public Observable createObservableCommand(Collection> requests) { + HystrixCommand command = self.createCommand(requests); + + // mark the number of requests being collapsed together + command.markAsCollapsedCommand(requests.size()); + + return command.toObservable(); + } + + @Override + public void mapResponseToRequests(BatchReturnType batchResponse, Collection> requests) { + self.mapResponseToRequests(batchResponse, requests); + } + + @Override + public HystrixCollapserKey getCollapserKey() { + return self.getCollapserKey(); + } + + }; + } + + private HystrixCollapserProperties getProperties() { + return collapserFactory.getProperties(); } /** @@ -159,7 +182,7 @@ private HystrixCollapser(HystrixCollapserKey collapserKey, Scope scope, Collapse * @return {@link HystrixCollapserKey} identifying this {@link HystrixCollapser} instance */ public HystrixCollapserKey getCollapserKey() { - return collapserKey; + return collapserFactory.getCollapserKey(); } /** @@ -178,7 +201,7 @@ public HystrixCollapserKey getCollapserKey() { * @return {@link Scope} that collapsing should be performed within. */ public Scope getScope() { - return scope; + return collapserFactory.getScope(); } /** @@ -274,6 +297,103 @@ protected Collection> requests); + /** + * Used for asynchronous execution with a callback by subscribing to the {@link Observable}. + *

+ * This eagerly starts execution the same as {@link #queue()} and {@link #execute()}. + * A lazy {@link Observable} can be obtained from {@link #toObservable()}. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ * Use {@link #toObservable(rx.Scheduler)} to schedule the callback differently. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that executes and calls back with the result of of {@link HystrixCommand}{@code } execution after passing through {@link #mapResponseToRequests} + * to transform the {@code } into {@code } + */ + public Observable observe() { + // us a ReplaySubject to buffer the eagerly subscribed-to Observable + ReplaySubject subject = ReplaySubject.create(); + // eagerly kick off subscription + toObservable().subscribe(subject); + // return the subject that can be subscribed to later while the execution has already started + return subject; + } + + /** + * A lazy {@link Observable} that will execute when subscribed to. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that lazily executes and calls back with the result of of {@link HystrixCommand}{@code } execution after passing through + * {@link #mapResponseToRequests} to transform the {@code } into {@code } + */ + public Observable toObservable() { + // when we callback with the data we want to do the work + // on a separate thread than the one giving us the callback + return toObservable(Schedulers.threadPoolForComputation()); + } + + /** + * A lazy {@link Observable} that will execute when subscribed to. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @param observeOn + * The {@link Scheduler} to execute callbacks on. + * @return {@code Observable} that lazily executes and calls back with the result of of {@link HystrixCommand}{@code } execution after passing through + * {@link #mapResponseToRequests} to transform the {@code } into {@code } + */ + public Observable toObservable(Scheduler observeOn) { + + /* try from cache first */ + if (getProperties().requestCachingEnabled().get()) { + Observable fromCache = requestCache.get(getCacheKey()); + if (fromCache != null) { + /* mark that we received this response from cache */ + // TODO Add collapser metrics so we can capture this information + // we can't add it to the command metrics because the command can change each time (dynamic key for example) + // and we don't have access to it when responding from cache + // collapserMetrics.markResponseFromCache(); + return fromCache; + } + } + + RequestCollapser requestCollapser = collapserFactory.getRequestCollapser(collapserInstanceWrapper); + Observable response = requestCollapser.submitRequest(getRequestArgument()); + if (getProperties().requestCachingEnabled().get()) { + /* + * A race can occur here with multiple threads queuing but only one will be cached. + * This means we can have some duplication of requests in a thread-race but we're okay + * with having some inefficiency in duplicate requests in the same batch + * and then subsequent requests will retrieve a previously cached Observable. + * + * If this is an issue we can make a lazy-future that gets set in the cache + * then only the winning 'put' will be invoked to actually call 'submitRequest' + */ + Observable o = response.cache(); + Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + if (fromCache == null) { + response = o; + } else { + response = fromCache; + } + } + return response; + } + /** * Used for synchronous execution. *

@@ -299,6 +419,7 @@ public ResponseType execute() { // we don't know what kind of exception this is so create a generic message and throw a new HystrixRuntimeException String message = getClass().getSimpleName() + " HystrixCollapser failed while executing."; logger.debug(message, e); // debug only since we're throwing the exception and someone higher will do something with it + //TODO should this be made a HystrixRuntimeException? throw new RuntimeException(message, e); } } @@ -315,577 +436,53 @@ public ResponseType execute() { * within an ExecutionException.getCause() (thrown by {@link Future#get}) if an error occurs and a fallback cannot be retrieved */ public Future queue() { - RequestCollapser collapser = null; - - if (Scope.REQUEST == getScope()) { - collapser = getCollapserForUserRequest(); - } else if (Scope.GLOBAL == getScope()) { - collapser = getCollapserForGlobalScope(); - } else { - logger.warn("Invalid Scope: " + getScope() + " Defaulting to REQUEST scope."); - collapser = getCollapserForUserRequest(); - } - /* try from cache first */ - if (properties.requestCachingEnabled().get()) { - Future fromCache = requestCache.get(getCacheKey()); - if (fromCache != null) { - /* mark that we received this response from cache */ - // TODO Add collapser metrics so we can capture this information - // we can't add it to the command metrics because the command can change each time (dynamic key for example) - // and we don't have access to it when responding from cache - // collapserMetrics.markResponseFromCache(); - return fromCache; - } - } - Future response = collapser.submitRequest(getRequestArgument()); - if (properties.requestCachingEnabled().get()) { - /* - * A race can occur here with multiple threads queuing but only one will be cached. - * This means we can have some duplication of requests in a thread-race but we're okay - * with having some inefficiency in duplicate requests in the same batch - * and then subsequent requests will retrieve a previously cached Future. - * - * If this is an issue we can make a lazy-future that gets set in the cache - * then only the winning 'put' will be invoked to actually call 'submitRequest' - */ - requestCache.putIfAbsent(getCacheKey(), response); - } - return response; - } - - /** - * Static global cache of RequestCollapsers for Scope.GLOBAL - */ - // String is CollapserKey.name() (we can't use CollapserKey directly as we can't guarantee it implements hashcode/equals correctly) - private static ConcurrentHashMap> globalScopedCollapsers = new ConcurrentHashMap>(); - - @SuppressWarnings("unchecked") - private RequestCollapser getCollapserForGlobalScope() { - RequestCollapser collapser = globalScopedCollapsers.get(getCollapserKey().name()); - if (collapser != null) { - return (RequestCollapser) collapser; - } - // create new collapser using 'this' first instance as the one that will get cached for future executions ('this' is stateless so we can do that) - RequestCollapser newCollapser = new RequestCollapser(this, timer, concurrencyStrategy); - RequestCollapser existing = globalScopedCollapsers.putIfAbsent(getCollapserKey().name(), newCollapser); - if (existing == null) { - // we won - return newCollapser; - } else { - // we lost ... another thread beat us - // shutdown the one we created but didn't get stored - newCollapser.shutdown(); - // return the existing one - return (RequestCollapser) existing; - } - } - - /** - * Static global cache of RequestVariables with RequestCollapsers for Scope.REQUEST - */ - // String is HystrixCollapserKey.name() (we can't use HystrixCollapserKey directly as we can't guarantee it implements hashcode/equals correctly) - private static ConcurrentHashMap>> requestScopedCollapsers = new ConcurrentHashMap>>(); - - /* we are casting because the Map needs to be but we know it is for this thread */ - @SuppressWarnings("unchecked") - private RequestCollapser getCollapserForUserRequest() { - return (RequestCollapser) getRequestVariableForCommand(getCollapserKey()).get(concurrencyStrategy); + final Observable o = toObservable(); + return o.toBlockingObservable().toFuture(); } /** - * Lookup (or create and store) the RequestVariable for a given HystrixCollapserKey. - * - * @param key - * @return HystrixRequestVariableHolder - */ - @SuppressWarnings("unchecked") - private HystrixRequestVariableHolder> getRequestVariableForCommand(final HystrixCollapserKey key) { - HystrixRequestVariableHolder> requestVariable = requestScopedCollapsers.get(key.name()); - if (requestVariable == null) { - // create new collapser using 'this' first instance as the one that will get cached for future executions ('this' is stateless so we can do that) - @SuppressWarnings({ "rawtypes" }) - RequestCollapserRequestVariable newCollapser = new RequestCollapserRequestVariable(this, timer, concurrencyStrategy); - HystrixRequestVariableHolder> existing = requestScopedCollapsers.putIfAbsent(key.name(), newCollapser); - if (existing == null) { - // this thread won, so return the one we just created - requestVariable = newCollapser; - } else { - // another thread beat us (this should only happen when we have concurrency on the FIRST request for the life of the app for this HystrixCollapser class) - requestVariable = existing; - /* - * This *should* be okay to discard the created object without cleanup as the RequestVariable implementation - * should properly do lazy-initialization and only call initialValue() the first time get() is called. - * - * If it does not correctly follow this contract then there is a chance of a memory leak here. - */ - } - } - return requestVariable; - } - - /** - * Request scoped RequestCollapser that lives inside a RequestVariable. + * Key to be used for request caching. + *

+ * By default this returns null which means "do not cache". + *

+ * To enable caching override this method and return a string key uniquely representing the state of a command instance. *

- * This depends on the RequestVariable getting reset before each user request in NFFilter to ensure the RequestCollapser is new for each user request. + * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. + * + * @return String cacheKey or null if not to cache */ - private static final class RequestCollapserRequestVariable extends HystrixRequestVariableHolder> { - - /** - * NOTE: There is only 1 instance of this for the life of the app per HystrixCollapser instance. The state changes on each request via the initialValue()/get() methods. - *

- * Thus, do NOT put any instance variables in this class that are not static for all threads. - */ - - private RequestCollapserRequestVariable(final HystrixCollapser commandCollapser, final CollapserTimer timer, final HystrixConcurrencyStrategy concurrencyStrategy) { - super(new HystrixRequestVariableLifecycle>() { - @Override - public RequestCollapser initialValue() { - // this gets calls once per request per HystrixCollapser instance - return new RequestCollapser(commandCollapser, timer, concurrencyStrategy); - } - - @Override - public void shutdown(RequestCollapser currentCollapser) { - // shut down the RequestCollapser (the internal timer tasks) - if (currentCollapser != null) { - currentCollapser.shutdown(); - } - } - }); - } - - } - - private static class RequestBatch { - private final long creationTime = System.currentTimeMillis(); - private final HystrixCollapser commandCollapser; - private final ConcurrentLinkedQueue> requests = new ConcurrentLinkedQueue>(); - // use AtomicInteger to count so we can use ConcurrentLinkedQueue instead of LinkedBlockingQueue - private final AtomicInteger count = new AtomicInteger(0); - private final HystrixCollapserProperties properties; - private final int maxBatchSize; - private final CountDownLatch batchCompleted = new CountDownLatch(1); - private final AtomicBoolean batchStarted = new AtomicBoolean(); - - private ReentrantReadWriteLock batchLock = new ReentrantReadWriteLock(); - - private volatile BatchFutureWrapper batchFuture; - - public RequestBatch(HystrixCollapserProperties properties, HystrixCollapser commandCollapser, int maxBatchSize) { - this.properties = properties; - this.commandCollapser = commandCollapser; - this.maxBatchSize = maxBatchSize; - } - - /** - * @return Future if offer accepted, null if batch is full, already started or completed - */ - public Future offer(RequestArgumentType arg) { - /* if the batch is started we reject the offer */ - if (batchStarted.get()) { - return null; - } - - /* - * The 'read' just means non-exclusive even though we are writing. - */ - if (batchLock.readLock().tryLock()) { - try { - int s = count.incrementAndGet(); - if (s > maxBatchSize) { - return null; - } else { - CollapsedRequestFutureImpl f = new CollapsedRequestFutureImpl(this, arg); - requests.add(f); - return f; - } - } finally { - batchLock.readLock().unlock(); - } - } else { - return null; - } - } - - /** - * Collapsed requests are triggered for batch execution and the array of arguments is passed in. - *

- * IMPORTANT IMPLEMENTATION DETAILS => The expected contract (responsibilities) of this method implementation is: - *

- *

    - *
  • Do NOT block => Do the work on a separate worker thread. Do not perform inline otherwise it will block other requests.
    • - *
  • Set ALL CollapsedRequest response values => Set the response values T on each CollapsedRequest, even if the response is NULL otherwise the user thread waiting on the response will - * think a response was never received and will either block indefinitely or will timeout while waiting.
    • - *
- * - * @param args - */ - public void executeBatchIfNotAlreadyStarted() { - /* - * - check that we only execute once since there's multiple paths to do so (timer, waiting thread or max batch size hit) - * - close the gate so 'offer' can no longer be invoked and we turn those threads away so they create a new batch - */ - if (batchStarted.compareAndSet(false, true)) { - /* wait for 'offer' threads to finish before executing the batch so 'requests' is complete */ - batchLock.writeLock().lock(); - try { - // shard batches - Collection>> shards = commandCollapser.shardRequests(requests); - if (shards.size() == 1) { - // not sharded so we'll get the single BatchFutureWrapper and assign it to this batch - for (Collection> shardRequests : shards) { - try { - // create a new command to handle this batch of requests - HystrixCommand command = commandCollapser.createCommand(shardRequests); - - // mark the number of requests being collapsed together - command.markAsCollapsedCommand(shardRequests.size()); - - // set the future on all requests so they can wait on this command completing or correctly receive errors if it fails or times out - batchFuture = new NonShardedBatchFutureWrapper(command.queue(), commandCollapser, shardRequests); - } catch (Exception e) { - logger.error("Exception while creating and queueing command with batch.", e); - // if a failure occurs we want to pass that exception to all of the Futures that we've returned - for (CollapsedRequest request : shardRequests) { - try { - request.setException(e); - } catch (IllegalStateException e2) { - logger.debug("Failed trying to setException on CollapsedRequest", e2); - } - } - } - } - } else { - // sharded - List> futurePerShard = new ArrayList>(); - // for each shard (1 or more) create a command, queue it and connect the Futures - for (Collection> shardRequests : shards) { - try { - // create a new command to handle this batch of requests - HystrixCommand command = commandCollapser.createCommand(shardRequests); - - // mark the number of requests being collapsed together - command.markAsCollapsedCommand(shardRequests.size()); - - // set the future on all requests so they can wait on this command completing or correctly receive errors if it fails or times out - futurePerShard.add(new NonShardedBatchFutureWrapper(command.queue(), commandCollapser, shardRequests)); - } catch (Exception e) { - logger.error("Exception while creating and queueing command with batch.", e); - // if a failure occurs we want to pass that exception to all of the Futures that we've returned - for (CollapsedRequest request : shardRequests) { - try { - request.setException(e); - } catch (IllegalStateException e2) { - logger.debug("Failed trying to setException on CollapsedRequest", e2); - } - } - } - } - // wrap the list of shards in a single reference for the entire batch - batchFuture = new ShardedBatchFutureWrapper(futurePerShard); - } - - } catch (Exception e) { - logger.error("Exception while sharding requests.", e); - // same error handling as we do around the shards, but this is a wider net in case the shardRequest method fails - for (CollapsedRequest request : requests) { - try { - request.setException(e); - } catch (IllegalStateException e2) { - logger.debug("Failed trying to setException on CollapsedRequest", e2); - } - } - } finally { - batchLock.writeLock().unlock(); - batchCompleted.countDown(); - } - } - } - - public void awaitBatchCompletion(CollapsedRequest request) throws InterruptedException, ExecutionException { - if (batchCompleted.getCount() > 0) { - if (!batchStarted.get()) { - // batch is not started, so so if enough time has passed to do it before the Timer thread gets to it - long timeSinceCreation = System.currentTimeMillis() - creationTime; - // if we've passed the time - if (timeSinceCreation >= properties.timerDelayInMilliseconds().get()) { - // try to executeBatch (only one thread will win this) - executeBatchIfNotAlreadyStarted(); - } else { - // wait for batch to execute - if (!batchCompleted.await(properties.timerDelayInMilliseconds().get() - timeSinceCreation, TimeUnit.MILLISECONDS)) { - // timed-out before timer triggered so try to executeBatch (only one thread will win this) - executeBatchIfNotAlreadyStarted(); - } - } - } - - // catch any threads not working above for the work to complete - batchCompleted.await(); - } - - // Wait on Future completing and performing the mapResponse work. - // Passing in timeout values here are ignored since it goes to the underlying HystrixCommand - // The possible vulnerability would be mapResponseToRequests being bad code and being latent. - if (batchFuture != null) { - // it can be null if an error occurred executing the batch - batchFuture.awaitAndThenMapResponsesToRequests(request); - } - } - + protected String getCacheKey() { + return null; } /** - * Must be thread-safe since it exists within a ThreadVariable which is request-scoped and can be accessed from multiple threads. + * Clears all state. If new requests come in instances will be recreated and metrics started from scratch. */ - @ThreadSafe - private static class RequestCollapser { - - private final HystrixCollapser commandCollapser; - private final AtomicReference> batch = new AtomicReference>(); - private final AtomicReference> timerListenerReference = new AtomicReference>(); - private final AtomicBoolean timerListenerRegistered = new AtomicBoolean(); - private final CollapserTimer timer; - private final HystrixCollapserProperties properties; - private final HystrixConcurrencyStrategy concurrencyStrategy; - - /** - * @param maxRequestsInBatch - * Maximum number of requests to include in a batch. If request count hits this threshold it will result in batch executions earlier than the scheduled delay interval. - * @param timerDelayInMilliseconds - * Interval between batch executions. - * @param commandCollapser - */ - public RequestCollapser(HystrixCollapser commandCollapser, CollapserTimer timer, HystrixConcurrencyStrategy concurrencyStrategy) { - this.commandCollapser = commandCollapser; // the command with implementation of abstract methods we need - this.concurrencyStrategy = concurrencyStrategy; - this.properties = commandCollapser.properties; - this.timer = timer; - batch.set(new RequestBatch(properties, commandCollapser, properties.maxRequestsInBatch().get())); - } - - /** - * Submit a request to a batch. If the batch maxSize is hit trigger the batch immediately. - * - * @param arg - * @return - */ - public Future submitRequest(RequestArgumentType arg) { - /* - * We only want the timer ticking if there are actually things to do so we register it the first time something is added. - */ - if (!timerListenerRegistered.get() && timerListenerRegistered.compareAndSet(false, true)) { - /* schedule the collapsing task to be executed every x milliseconds (x defined inside CollapsedTask) */ - timerListenerReference.set(timer.addListener(new CollapsedTask())); - } - - // loop until succeed (compare-and-set spin-loop) - while (true) { - RequestBatch b = batch.get(); - Future f = b.offer(arg); - // it will always get a Future unless we hit the max batch size - if (f != null) { - return f; - } else { - // we hit max batch size so create a new batch and set it if another thread doesn't beat us - executeBatchAndCreateNew(b); - } - } - } - - private void executeBatchAndCreateNew(RequestBatch b) { - if (batch.compareAndSet(b, new RequestBatch(properties, commandCollapser, properties.maxRequestsInBatch().get()))) { - // this thread won so trigger the previous batch - b.executeBatchIfNotAlreadyStarted(); - } - } - - /** - * Called from RequestVariable.shutdown() to unschedule the task. - */ - public void shutdown() { - Collection> requests = batch.get().requests; - if (requests.size() > 0) { - logger.warn("Requests still exist in queue but will not be executed due to RequestCollapser shutdown: " + requests.size(), new IllegalStateException()); - /* - * In the event that there is a concurrency bug or thread scheduling prevents the timer from ticking we need to handle this so the Future.get() calls do not block. - * - * I haven't been able to reproduce this use case on-demand but when stressing a machine saw this occur briefly right after the JVM paused (logs stopped scrolling). - * - * This safety-net just prevents the CollapsedRequestFutureImpl.get() from waiting on the CountDownLatch until its max timeout. - */ - for (CollapsedRequest request : requests) { - try { - request.setException(new IllegalStateException("Requests not executed before shutdown.")); - } catch (Exception e) { - logger.debug("Failed to setException on CollapsedRequestFutureImpl instances.", e); - } - /** - * https://github.com/Netflix/Hystrix/issues/78 Include more info when collapsed requests remain in queue - */ - logger.warn("Request still in queue but not be executed due to RequestCollapser shutdown. Argument => " + request.getArgument() + " Request Object => " + request, new IllegalStateException()); - } - - } - if (timerListenerReference.get() != null) { - // if the timer was started we'll clear it - timerListenerReference.get().clear(); - } - } - - /** - * Executed on each Timer interval to drain the queue and execute the batch command. - */ - private class CollapsedTask implements TimerListener { - final Callable callableWithContextOfParent; - - CollapsedTask() { - // this gets executed from the context of a HystrixCommand parent thread (such as a Tomcat thread) - // so we create the callable now where we can capture the thread context - callableWithContextOfParent = concurrencyStrategy.wrapCallable(new HystrixContextCallable(new Callable() { - // the wrapCallable call allows a strategy to capture thread-context if desired - - @Override - public Void call() throws Exception { - try { - // do execution within context of wrapped Callable - executeBatchAndCreateNew(batch.get()); - } catch (Throwable t) { - logger.error("Error occurred trying to executeRequestsFromQueue.", t); - // ignore error so we don't kill the Timer mainLoop and prevent further items from being scheduled - // http://jira.netflix.com/browse/API-5042 HystrixCommand: Collapser TimerThread Vulnerable to Shutdown - } - return null; - } - - })); - } - - @Override - public void tick() { - - // don't bother if we don't have any requests queued up - if (batch.get().requests.size() > 0) { - // this gets executed from the context of the CollapserTimer thread - try { - callableWithContextOfParent.call(); - } catch (Exception e) { - logger.error("Error occurred trying to execute callable inside CollapsedTask from Timer.", e); - e.printStackTrace(); - } - } - } - - @Override - public int getIntervalTimeInMilliseconds() { - return properties.timerDelayInMilliseconds().get(); - } - - } - - } - - private static interface BatchFutureWrapper { - public void awaitAndThenMapResponsesToRequests(CollapsedRequest collapsedRequest) throws InterruptedException; - } - - private static class NonShardedBatchFutureWrapper implements BatchFutureWrapper { - private final Future actualFuture; - private final HystrixCollapser command; - private final Collection> requests; - private final CountDownLatch isCompleted = new CountDownLatch(1); - private AtomicBoolean mapResponseWork = new AtomicBoolean(false); - - private NonShardedBatchFutureWrapper(Future actualFuture, HystrixCollapser command, Collection> requests) { - this.actualFuture = actualFuture; - this.command = command; - this.requests = requests; - } - - public void awaitAndThenMapResponsesToRequests(CollapsedRequest collapsedRequest) throws InterruptedException { - /* only one thread should do this and all the rest will proceed to actualFuture.get() */ - if (mapResponseWork.compareAndSet(false, true)) { - try { - try { - /* we only want one thread to execute the above code */ - command.mapResponseToRequests(actualFuture.get(), requests); - } catch (Exception e) { - logger.error("Exception mapping responses to requests.", e); - // if a failure occurs we want to pass that exception to all of the Futures that we've returned - for (CollapsedRequest request : requests) { - try { - if (((CollapsedRequestFutureImpl) request).responseReference.get() == null) { - request.setException(e); - } - } catch (IllegalStateException e2) { - // if we have partial responses set in mapResponseToRequests - // then we may get IllegalStateException as we loop over them - // so we'll log but continue to the rest - logger.error("Partial success of 'mapResponseToRequests' resulted in IllegalStateException while setting Exception. Continuing ... ", e2); - } - } - } - - // check that all requests had setResponse or setException invoked in case 'mapResponseToRequests' was implemented poorly - for (CollapsedRequest request : requests) { - try { - if (((CollapsedRequestFutureImpl) request).responseReference.get() == null) { - request.setException(new IllegalStateException("No response set by " + command.getCollapserKey().name() + " 'mapResponseToRequests' implementation.")); - } - } catch (IllegalStateException e2) { - logger.debug("Partial success of 'mapResponseToRequests' resulted in IllegalStateException while setting 'No response set' Exception. Continuing ... ", e2); - } - } - } finally { - // release all threads waiting on mapResponseToRequests being done - isCompleted.countDown(); - } - } - - // all other threads will wait until the block above is completed - isCompleted.await(); - } - + /* package */static void reset() { + RequestCollapserFactory.reset(); } - private static class ShardedBatchFutureWrapper implements BatchFutureWrapper { - private final List> shards; - - public ShardedBatchFutureWrapper(List> shards) { - this.shards = shards; - } - - @Override - public void awaitAndThenMapResponsesToRequests(CollapsedRequest collapsedRequest) throws InterruptedException { - // TODO have a map of collapsedRequest to shard so it only blocks on it, not all - for (BatchFutureWrapper shard : shards) { - shard.awaitAndThenMapResponsesToRequests(collapsedRequest); - } + private static String getDefaultNameFromClass(@SuppressWarnings("rawtypes") Class cls) { + String fromCache = defaultNameCache.get(cls); + if (fromCache != null) { + return fromCache; } - - } - - private static interface CollapserTimer { - - public Reference addListener(TimerListener collapseTask); - - } - - private static class RealCollapserTimer implements CollapserTimer { - /* single global timer that all collapsers will schedule their tasks on */ - private final static HystrixTimer timer = HystrixTimer.getInstance(); - - @Override - public Reference addListener(TimerListener collapseTask) { - return timer.addTimerListener(collapseTask); + // generate the default + // default HystrixCommandKey to use if the method is not overridden + String name = cls.getSimpleName(); + if (name.equals("")) { + // we don't have a SimpleName (anonymous inner class) so use the full class name + name = cls.getName(); + name = name.substring(name.lastIndexOf('.') + 1, name.length()); } - + defaultNameCache.put(cls, name); + return name; } /** * A request argument RequestArgumentType that was collapsed for batch processing and needs a response ResponseType set on it by the executeBatch implementation. */ - public static interface CollapsedRequest { + public interface CollapsedRequest { /** * The request argument passed into the {@link HystrixCollapser} instance constructor which was then collapsed. * @@ -913,215 +510,6 @@ public static interface CollapsedRequest { public void setException(Exception exception); } - /* - * Private implementation class that combines the Future and CollapsedRequest functionality. - *

- * We expose these via interfaces only since we want clients to only see Future and implementors to only see CollapsedRequest, not the combination of the two. - * - * @param - * - * @param - */ - private static class CollapsedRequestFutureImpl implements CollapsedRequest, Future { - private final R argument; - private final AtomicReference> responseReference = new AtomicReference>(); - private final RequestBatch batch; - - public CollapsedRequestFutureImpl(RequestBatch batch, R arg) { - this.argument = arg; - this.batch = batch; - } - - /** - * The request argument. - * - * @return request argument - */ - @Override - public R getArgument() { - return argument; - } - - /** - * When set any client thread blocking on get() will immediately be unblocked and receive the response. - * - * @throws IllegalStateException - * if called more than once or after setException. - * @param response - */ - @Override - public void setResponse(T response) { - /* only set it if null */ - boolean didSet = responseReference.compareAndSet(null, new ResponseHolder(response, null)); - // if it was already set to an exception, then throw an IllegalStateException as the developer should not be trying to set both - if (!didSet || responseReference.get().getException() != null) { - throw new IllegalStateException("Response or Exception has already been set."); - } - } - - /** - * When set any client thread blocking on get() will immediately be unblocked and receive the exception. - * - * @throws IllegalStateException - * if called more than once or after setResponse. - * @param response - */ - @Override - public void setException(Exception e) { - /* only set it if null */ - boolean didSet = responseReference.compareAndSet(null, new ResponseHolder(null, e)); - // if it was already set to a response, then throw an IllegalStateException as the developer should not be trying to set both - if (!didSet || responseReference.get().getResponse() != null) { - throw new IllegalStateException("Response or Exception has already been set."); - } - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - throw new IllegalStateException("We don't support cancelling tasks submitted for batch execution."); - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return responseReference.get() != null; - } - - @Override - public T get() throws InterruptedException, ExecutionException { - return getValueFromBatch(); - } - - @Override - public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return getValueFromBatch(); - } - - public T getValueFromBatch() throws InterruptedException, ExecutionException { - try { - // wait for completion or throw exception - batch.awaitBatchCompletion(this); - if (responseReference.get() == null) { - /* - * https://github.com/Netflix/Hystrix/issues/80 - * This should never happen because mapResponseToRequests checks for NULLs and calls setException. - * If this happens it means we have a concurrency bug somewhere - */ - throw new ExecutionException("Null ResponseReference", new IllegalStateException("ResponseReference is NULL. Please file a bug at https://github.com/Netflix/Hystrix/issues")); - } else { - // we got past here so let's return the response now - if (responseReference.get().getException() != null) { - throw new ExecutionException(responseReference.get().getException()); - } - return responseReference.get().getResponse(); - } - } catch (ExecutionException e) { - logger.error("ExecutionException on CollapsedRequest.get", e); - throw e; - } - } - - /** - * Used for atomic compound updates. - */ - private static class ResponseHolder { - private final T response; - private final Exception e; - - public ResponseHolder(T response, Exception e) { - this.response = response; - this.e = e; - } - - public T getResponse() { - return response; - } - - public Exception getException() { - return e; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((e == null) ? 0 : e.hashCode()); - result = prime * result + ((response == null) ? 0 : response.hashCode()); - return result; - } - - @SuppressWarnings("rawtypes") - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ResponseHolder other = (ResponseHolder) obj; - if (e == null) { - if (other.e != null) - return false; - } else if (!e.equals(other.e)) - return false; - if (response == null) { - if (other.response != null) - return false; - } else if (!response.equals(other.response)) - return false; - return true; - } - - } - } - - /** - * Key to be used for request caching. - *

- * By default this returns null which means "do not cache". - *

- * To enable caching override this method and return a string key uniquely representing the state of a command instance. - *

- * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. - * - * @return String cacheKey or null if not to cache - */ - protected String getCacheKey() { - return null; - } - - /** - * Clears all state. If new requests come in instances will be recreated and metrics started from scratch. - */ - /* package */static void reset() { - defaultNameCache.clear(); - globalScopedCollapsers.clear(); - requestScopedCollapsers.clear(); - HystrixTimer.reset(); - } - - private static String getDefaultNameFromClass(@SuppressWarnings("rawtypes") Class cls) { - String fromCache = defaultNameCache.get(cls); - if (fromCache != null) { - return fromCache; - } - // generate the default - // default HystrixCommandKey to use if the method is not overridden - String name = cls.getSimpleName(); - if (name.equals("")) { - // we don't have a SimpleName (anonymous inner class) so use the full class name - name = cls.getName(); - name = name.substring(name.lastIndexOf('.') + 1, name.length()); - } - defaultNameCache.put(cls, name); - return name; - } - /** * Fluent interface for arguments to the {@link HystrixCollapser} constructor. *

@@ -1197,8 +585,7 @@ public static class UnitTests { public void init() { counter.set(0); // since we're going to modify properties of the same class between tests, wipe the cache each time - requestScopedCollapsers.clear(); - globalScopedCollapsers.clear(); + reset(); /* we must call this to simulate a new request lifecycle running and clearing caches */ HystrixRequestContext.initializeContext(); } @@ -1296,65 +683,21 @@ public void testRequestsOverTime() throws Exception { assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); } - /** - * Check when the Timer is latent that the Future.get() will trigger the batch execution. - * - * @throws Exception - */ @Test - public void testBatchExecutesViaGetIfTimerDoesntFire() throws Exception { + public void testUnsubscribeOnOneDoesntKillBatch() throws Exception { TestCollapserTimer timer = new TestCollapserTimer(); Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - // purposefully don't increment the timer - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); + // kill the first + response1.cancel(true); - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Check when the Timer is latent that the Future.get() will trigger the batch execution. - * - * @throws Exception - */ - @Test - public void testBatchExecutesViaGetIfTimerDoesntFireMultiThreaded() throws Exception { - final TestCollapserTimer timer = new TestCollapserTimer(); - final AtomicReference v1 = new AtomicReference(); - final AtomicReference v2 = new AtomicReference(); - - Thread t1 = new Thread(new HystrixContextRunnable(new Runnable() { - - @Override - public void run() { - v1.set(new TestRequestCollapser(timer, counter, 1).execute()); - } - - })); - Thread t2 = new Thread(new HystrixContextRunnable(new Runnable() { - - @Override - public void run() { - v2.set(new TestRequestCollapser(timer, counter, 2).execute()); - } - - })); - - t1.start(); - t2.start(); - - // purposefully don't increment the timer - - t1.join(); - t2.join(); + timer.incrementTime(10); // let time pass that equals the default delay/period - assertEquals("1", v1.get()); - assertEquals("2", v2.get()); + // the first is cancelled so should return null + assertEquals(null, response1.get()); + // we should still get a response on the second + assertEquals("2", response2.get()); assertEquals(1, counter.get()); @@ -1387,7 +730,7 @@ public void testRequestScope() throws Exception { Future response2 = new TestRequestCollapser(timer, counter, "2").queue(); // simulate a new request - requestScopedCollapsers.clear(); + RequestCollapserFactory.resetRequest(); Future response3 = new TestRequestCollapser(timer, counter, "3").queue(); Future response4 = new TestRequestCollapser(timer, counter, "4").queue(); @@ -1411,7 +754,7 @@ public void testGlobalScope() throws Exception { Future response2 = new TestGloballyScopedRequestCollapser(timer, counter, "2").queue(); // simulate a new request - requestScopedCollapsers.clear(); + RequestCollapserFactory.resetRequest(); Future response3 = new TestGloballyScopedRequestCollapser(timer, counter, "3").queue(); Future response4 = new TestGloballyScopedRequestCollapser(timer, counter, "4").queue(); @@ -1519,7 +862,7 @@ public void testRequestVariableLifecycle1() throws Exception { System.out.println("timer.tasks.size() B: " + timer.tasks.size()); - HystrixRequestVariableHolder> rv = requestScopedCollapsers.get(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); + HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); assertNotNull(rv); // they should have all been removed as part of ThreadContext.remove() @@ -1594,7 +937,7 @@ public void run() { // simulate request lifecycle requestContext.shutdown(); - HystrixRequestVariableHolder> rv = requestScopedCollapsers.get(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); + HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); assertNotNull(rv); // they should have all been removed as part of ThreadContext.remove() @@ -2023,7 +1366,7 @@ public String getRequestArgument() { } @Override - public HystrixCommand> createCommand(final Collection> requests) { + public HystrixCommand> createCommand(final Collection> requests) { /* return a mocked command */ HystrixCommand> command = new TestCollapserCommand(requests); if (commandsExecuted != null) { @@ -2040,7 +1383,7 @@ public void mapResponseToRequests(List batchResponse, Collection " + batchResponse.size() + " : " + requests.size()); } int i = 0; for (CollapsedRequest request : requests) { @@ -2061,11 +1404,11 @@ public TestShardedRequestCollapser(TestCollapserTimer timer, AtomicInteger count } @Override - protected Collection>> shardRequests(Collection> requests) { - Collection> typeA = new ArrayList>(); - Collection> typeB = new ArrayList>(); + protected Collection>> shardRequests(Collection> requests) { + Collection> typeA = new ArrayList>(); + Collection> typeB = new ArrayList>(); - for (HystrixCollapser.CollapsedRequest request : requests) { + for (CollapsedRequest request : requests) { if (request.getArgument().endsWith("a")) { typeA.add(request); } else if (request.getArgument().endsWith("b")) { @@ -2073,7 +1416,7 @@ protected Collection>> shards = new ArrayList>>(); + ArrayList>> shards = new ArrayList>>(); shards.add(typeA); shards.add(typeB); return shards; @@ -2102,7 +1445,7 @@ public TestRequestCollapserWithFaultyCreateCommand(TestCollapserTimer timer, Ato } @Override - public HystrixCommand> createCommand(Collection> requests) { + public HystrixCommand> createCommand(Collection> requests) { throw new RuntimeException("some failure"); } @@ -2118,7 +1461,7 @@ public TestRequestCollapserWithShortCircuitedCommand(TestCollapserTimer timer, A } @Override - public HystrixCommand> createCommand(Collection> requests) { + public HystrixCommand> createCommand(Collection> requests) { // args don't matter as it's short-circuited return new ShortCircuitedCommand(); } @@ -2135,7 +1478,7 @@ public TestRequestCollapserWithFaultyMapToResponse(TestCollapserTimer timer, Ato } @Override - public void mapResponseToRequests(List batchResponse, Collection> requests) { + public void mapResponseToRequests(List batchResponse, Collection> requests) { // pretend we blow up with an NPE throw new NullPointerException("batchResponse was null and we blew up"); } @@ -2153,6 +1496,7 @@ private static class TestCollapserCommand extends TestHystrixCommand run() { + System.out.println(">>> TestCollapserCommand run() ... batch size: " + requests.size()); // simulate a batch request ArrayList response = new ArrayList(); for (CollapsedRequest request : requests) { diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java index 8d8ebedd1..712c9483d 100755 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java @@ -18,13 +18,14 @@ import static org.junit.Assert.*; import java.io.IOException; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -36,7 +37,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.concurrent.NotThreadSafe; @@ -48,6 +48,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import rx.Observable; +import rx.Observer; +import rx.Scheduler; +import rx.Subscription; +import rx.concurrency.Schedulers; +import rx.operators.AtomicObservableSubscription; +import rx.subjects.ReplaySubject; +import rx.subscriptions.Subscriptions; +import rx.util.functions.Action0; +import rx.util.functions.Func1; +import rx.util.functions.Func2; + import com.netflix.config.ConfigurationManager; import com.netflix.hystrix.HystrixCircuitBreaker.NoOpCircuitBreaker; import com.netflix.hystrix.HystrixCircuitBreaker.TestCircuitBreaker; @@ -69,6 +81,8 @@ import com.netflix.hystrix.strategy.properties.HystrixProperty; import com.netflix.hystrix.util.ExceptionThreadingUtility; import com.netflix.hystrix.util.HystrixRollingNumberEvent; +import com.netflix.hystrix.util.HystrixTimer; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; /** * Used to wrap code that will execute potentially risky functionality (typically meaning a service call over the network) @@ -111,8 +125,10 @@ public abstract class HystrixCommand implements HystrixExecutable { private static final ConcurrentHashMap executionSemaphorePerCircuit = new ConcurrentHashMap(); /* END EXECUTION Semaphore */ - /* used to track whenever the user invokes the command using execute(), queue() or fireAndForget() ... also used to know if execution has begun */ - private AtomicLong invocationStartTime = new AtomicLong(-1); + private final AtomicReference> timeoutTimer = new AtomicReference>(); + + private AtomicBoolean started = new AtomicBoolean(); + private volatile long invocationStartTime = -1; /** * Instance of RequestCache logic @@ -148,6 +164,9 @@ protected HystrixCommand(HystrixCommandGroupKey group) { * NOTE: The {@link HystrixCommandKey} is used to associate a {@link HystrixCommand} with {@link HystrixCircuitBreaker}, {@link HystrixCommandMetrics} and other objects. *

* Do not create multiple {@link HystrixCommand} implementations with the same {@link HystrixCommandKey} but different injected default properties as the first instantiated will win. + *

+ * Properties passed in via {@link Setter#andCommandPropertiesDefaults} or {@link Setter#andThreadPoolPropertiesDefaults} are cached for the given {@link HystrixCommandKey} for the life of the JVM + * or until {@link Hystrix#reset()} is called. Dynamic properties allow runtime changes. Read more on the Hystrix Wiki. * * @param setter * Fluent interface for constructor arguments @@ -394,239 +413,668 @@ public HystrixCommandProperties getProperties() { * if a failure occurs and a fallback cannot be retrieved * @throws HystrixBadRequestException * if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once */ public R execute() { try { - /* used to track userThreadExecutionTime */ - if (!invocationStartTime.compareAndSet(-1, System.currentTimeMillis())) { - throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); - } + return queue().get(); + } catch (Exception e) { + throw decomposeException(e); + } + } + + /** + * Used for asynchronous execution of command. + *

+ * This will queue up the command on the thread pool and return an {@link Future} to get the result once it completes. + *

+ * NOTE: If configured to not run in a separate thread, this will have the same effect as {@link #execute()} and will block. + *

+ * We don't throw an exception but just flip to synchronous execution so code doesn't need to change in order to switch a command from running on a separate thread to the calling thread. + * + * @return {@code Future} Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Future.get()} in {@link ExecutionException#getCause()} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Future.get()} in {@link ExecutionException#getCause()} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Future queue() { + /* + * --- Schedulers.immediate() + * + * We use the 'immediate' schedule since Future.get() is blocking so we don't want to bother doing the callback to the Future on a separate thread + * as we don't need to separate the Hystrix thread from user threads since they are already providing it via the Future.get() call. + * + * --- performAsyncTimeout: false + * + * We pass 'false' to tell the Observable we will block on it so it doesn't schedule an async timeout. + * + * This optimizes for using the calling thread to do the timeout rather than scheduling another thread. + * + * In a tight-loop of executing commands this optimization saves a few microseconds per execution. + * It also just makes no sense to use a separate thread to timeout the command when the calling thread + * is going to sit waiting on it. + */ + final ObservableCommand o = toObservable(Schedulers.immediate(), false); + final Future f = o.toBlockingObservable().toFuture(); + + /* special handling of error states that throw immediately */ + if (f.isDone()) { try { - /* try from cache first */ - if (isRequestCachingEnabled()) { - Future fromCache = requestCache.get(getCacheKey()); - if (fromCache != null) { - /* mark that we received this response from cache */ - metrics.markResponseFromCache(); - return asCachedFuture(fromCache).get(); + f.get(); + return f; + } catch (Exception e) { + RuntimeException re = decomposeException(e); + if (re instanceof HystrixRuntimeException) { + HystrixRuntimeException hre = (HystrixRuntimeException) re; + if (hre.getFailureType() == FailureType.COMMAND_EXCEPTION || hre.getFailureType() == FailureType.TIMEOUT) { + // we don't throw these types from queue() only from queue().get() as they are execution errors + return f; + } else { + // these are errors we throw from queue() as they as rejection type errors + throw hre; } + } else { + throw re; } + } + } - // mark that we're starting execution on the ExecutionHook - executionHook.onStart(this); + return new Future() { - /* determine if we're allowed to execute */ - if (!circuitBreaker.allowRequest()) { - // record that we are returning a short-circuited fallback - metrics.markShortCircuited(); - // short-circuit and go directly to fallback - return getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, "short-circuited"); - } + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return f.cancel(mayInterruptIfRunning); + } - try { - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - // we want to run in a separate thread with timeout protection - return queueInThread().get(); - } else { - return executeWithSemaphore(); - } - } catch (RuntimeException e) { - // count that we're throwing an exception and rethrow - metrics.markExceptionThrown(); - throw e; - } + @Override + public boolean isCancelled() { + return f.isCancelled(); + } - } catch (Exception e) { - if (e instanceof HystrixBadRequestException) { - throw (HystrixBadRequestException) e; - } - if (e.getCause() instanceof HystrixBadRequestException) { - throw (HystrixBadRequestException) e.getCause(); - } - if (e instanceof HystrixRuntimeException) { - throw (HystrixRuntimeException) e; - } - // if we have an exception we know about we'll throw it directly without the wrapper exception - if (e.getCause() instanceof HystrixRuntimeException) { - throw (HystrixRuntimeException) e.getCause(); - } - // we don't know what kind of exception this is so create a generic message and throw a new HystrixRuntimeException - String message = getLogMessagePrefix() + " failed while executing."; - logger.debug(message, e); // debug only since we're throwing the exception and someone higher will do something with it - throw new HystrixRuntimeException(FailureType.COMMAND_EXCEPTION, this.getClass(), message, e, null); + @Override + public boolean isDone() { + return f.isDone(); } - } finally { - recordExecutedCommand(); - } - } - private R executeWithSemaphore() { - TryableSemaphore executionSemaphore = getExecutionSemaphore(); - // acquire a permit - if (executionSemaphore.tryAcquire()) { - try { - // store the command that is being run - Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); - // we want to run it synchronously - R response = executeCommand(); - response = executionHook.onComplete(this, response); - // put in cache - if (isRequestCachingEnabled()) { - requestCache.putIfAbsent(getCacheKey(), asFutureForCache(response)); + @Override + public R get() throws InterruptedException, ExecutionException { + return performBlockingGetWithTimeout(o, f); + } + + /** + * --- Non-Blocking Timeout (performAsyncTimeout:true) --- + * + * When 'toObservable' is done with non-blocking timeout then timeout functionality is provided + * by a separate HystrixTimer thread that will "tick" and cancel the underlying async Future inside the Observable. + * + * This method allows stealing that responsibility and letting the thread that's going to block anyways + * do the work to reduce pressure on the HystrixTimer. + * + * Blocking via queue().get() on a non-blocking timeout will work it's just less efficient + * as it involves an extra thread and cancels the scheduled action that does the timeout. + * + * --- Blocking Timeout (performAsyncTimeout:false) --- + * + * When blocking timeout is assumed (default behavior for execute/queue flows) then the async + * timeout will not have been scheduled and this will wait in a blocking manner and if a timeout occurs + * trigger the timeout logic that comes from inside the Observable/Observer. + * + * + * --- Examples + * + * Stack for timeout with performAsyncTimeout=false (note the calling thread via get): + * + * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:788) + * at com.netflix.hystrix.HystrixCommand$1.performBlockingGetWithTimeout(HystrixCommand.java:536) + * at com.netflix.hystrix.HystrixCommand$1.get(HystrixCommand.java:484) + * at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:413) + * + * + * Stack for timeout with performAsyncTimeout=true (note the HystrixTimer involved): + * + * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:799) + * at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:101) + * + * + * + * @param o + * @param f + * @throws InterruptedException + * @throws ExecutionException + */ + protected R performBlockingGetWithTimeout(final ObservableCommand o, final Future f) throws InterruptedException, ExecutionException { + // shortcut if already done + if (f.isDone()) { + return f.get(); } - /* - * We don't bother looking for whether someone else also put it in the cache since we've already executed and received a response. - * In this path we are synchronous so don't have the option of queuing a Future. + + // it's still working so proceed with blocking/timeout logic + HystrixCommand originalCommand = o.getCommand(); + /** + * One thread will get the timeoutTimer if it's set and clear it then do blocking timeout. + *

+ * If non-blocking timeout was scheduled this will unschedule it. If it wasn't scheduled it is basically + * a no-op but fits the same interface so blocking and non-blocking flows both work the same. + *

+ * This "originalCommand" concept exists because of request caching. We only do the work and timeout logic + * on the original, not the cached responses. However, whichever the first thread is that comes in to block + * will be the one who performs the timeout logic. + *

+ * If request caching is disabled then it will always go into here. */ - return response; - } finally { - executionSemaphore.release(); + if (originalCommand != null) { + Reference timer = originalCommand.timeoutTimer.getAndSet(null); + if (timer != null) { + /** + * If an async timeout was scheduled then: + * + * - We are going to clear the Reference so the scheduler threads stop managing the timeout + * and we'll take over instead since we're going to be blocking on it anyways. + * + * - Other threads (since we won the race) will just wait on the normal Future which will release + * once the Observable is marked as completed (which may come via timeout) + * + * If an async timeout was not scheduled: + * + * - We go through the same flow as we receive the same interfaces just the "timer.clear()" will do nothing. + */ + // get the timer we'll use to perform the timeout + TimerListener l = timer.get(); + // remove the timer from the scheduler + timer.clear(); + + // determine how long we should wait for, taking into account time since work started + // and when this thread came in to block. If invocationTime hasn't been set then assume time remaining is entire timeout value + // as this maybe a case of multiple threads trying to run this command in which one thread wins but even before the winning thread is able to set + // the starttime another thread going via the Cached command route gets here first. + long timeout = originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); + long timeRemaining = timeout; + long currTime = System.currentTimeMillis(); + if (originalCommand.invocationStartTime != -1) { + timeRemaining = (originalCommand.invocationStartTime + + originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get()) + - currTime; - // pop the command that is being run - Hystrix.endCurrentThreadExecutingCommand(); + } + if (timeRemaining > 0) { + // we need to block with the calculated timeout + try { + return f.get(timeRemaining, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + if (l != null) { + // this perform the timeout logic on the Observable/Observer + l.tick(); + } + } + } else { + // this means it should have already timed out so do so if it is not completed + if (!f.isDone()) { + if (l != null) { + l.tick(); + } + } + } + } + } + // other threads will block until the "l.tick" occurs and releases the underlying Future. + return f.get(); + } - /* execution time on execution via semaphore */ - recordTotalExecutionTime(invocationStartTime.get()); + @Override + public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return get(); } - } else { - // mark on counter - metrics.markSemaphoreRejection(); - logger.debug("HystrixCommand Execution Rejection by Semaphore"); // debug only since we're throwing the exception and someone higher will do something with it - return getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, "could not acquire a semaphore for execution"); + + }; + + } + + /** + * Take an Exception and determine whether to throw it, its cause or a new HystrixRuntimeException. + *

+ * This will only throw an HystrixRuntimeException, HystrixBadRequestException or IllegalStateException + * + * @param e + * @return HystrixRuntimeException, HystrixBadRequestException or IllegalStateException + */ + protected RuntimeException decomposeException(Exception e) { + if (e instanceof IllegalStateException) { + return (IllegalStateException) e; + } + if (e instanceof HystrixBadRequestException) { + return (HystrixBadRequestException) e; + } + if (e.getCause() instanceof HystrixBadRequestException) { + return (HystrixBadRequestException) e.getCause(); + } + if (e instanceof HystrixRuntimeException) { + return (HystrixRuntimeException) e; } + // if we have an exception we know about we'll throw it directly without the wrapper exception + if (e.getCause() instanceof HystrixRuntimeException) { + return (HystrixRuntimeException) e.getCause(); + } + // we don't know what kind of exception this is so create a generic message and throw a new HystrixRuntimeException + String message = getLogMessagePrefix() + " failed while executing."; + logger.debug(message, e); // debug only since we're throwing the exception and someone higher will do something with it + return new HystrixRuntimeException(FailureType.COMMAND_EXCEPTION, this.getClass(), message, e, null); } /** - * Used for asynchronous execution of command. + * Used for asynchronous execution of command with a callback by subscribing to the {@link Observable}. *

- * This will queue up the command on the thread pool and return an {@link Future} to get the result once it completes. + * This eagerly starts execution of the command the same as {@link #queue()} and {@link #execute()}. + * A lazy {@link Observable} can be obtained from {@link #toObservable()}. *

- * NOTE: If configured to not run in a separate thread, this will have the same effect as {@link #execute()} and will block. + * Callback Scheduling *

- * We don't throw an exception but just flip to synchronous execution so code doesn't need to change in order to switch a command from running on a separate thread to the calling thread. + *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ * Use {@link #toObservable(rx.Scheduler)} to schedule the callback differently. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. * - * @return {@code Future} Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @return {@code Observable} that executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. * @throws HystrixRuntimeException * if a fallback does not exist *

*

    - *
  • via {@code Future.get()} in {@link ExecutionException#getCause()} if a failure occurs
    • - *
  • or immediately if the command can not be queued (such as short-circuited or thread-pool/semaphore rejected)
    • + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • *
* @throws HystrixBadRequestException - * via {@code Future.get()} in {@link ExecutionException#getCause()} if invalid arguments or state were used representing a user failure, not a system failure + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once */ - public Future queue() { - try { - /* used to track userThreadExecutionTime */ - if (!invocationStartTime.compareAndSet(-1, System.currentTimeMillis())) { - throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); - } - if (isRequestCachingEnabled()) { - /* try from cache first */ - Future fromCache = requestCache.get(getCacheKey()); - if (fromCache != null) { - /* mark that we received this response from cache */ - metrics.markResponseFromCache(); - return asCachedFuture(fromCache); - } - } + public Observable observe() { + // us a ReplaySubject to buffer the eagerly subscribed-to Observable + ReplaySubject subject = ReplaySubject.create(); + // eagerly kick off subscription + toObservable().subscribe(subject); + // return the subject that can be subscribed to later while the execution has already started + return subject; + } + + /** + * A lazy {@link Observable} that will execute the command when subscribed to. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that lazily executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable toObservable() { + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + return toObservable(Schedulers.threadPoolForComputation()); + } else { + // semaphore isolation is all blocking, no new threads involved + // so we'll use the calling thread + return toObservable(Schedulers.immediate()); + } + } + + /** + * A lazy {@link Observable} that will execute the command when subscribed to. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @param observeOn + * The {@link Scheduler} to execute callbacks on. + * @return {@code Observable} that lazily executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable toObservable(Scheduler observeOn) { + return toObservable(observeOn, true); + } - // mark that we're starting execution on the ExecutionHook - executionHook.onStart(this); + private ObservableCommand toObservable(Scheduler observeOn, boolean performAsyncTimeout) { + /* this is a stateful object so can only be used once */ + if (!started.compareAndSet(false, true)) { + throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); + } - /* determine if we're allowed to execute */ - if (!circuitBreaker.allowRequest()) { - // record that we are returning a short-circuited fallback - metrics.markShortCircuited(); - // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) - return asFuture(getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, "short-circuited")); + /* try from cache first */ + if (isRequestCachingEnabled()) { + Observable fromCache = requestCache.get(getCacheKey()); + if (fromCache != null) { + /* mark that we received this response from cache */ + metrics.markResponseFromCache(); + return new CachedObservableResponse((CachedObservableOriginal) fromCache, this); } + } - /* nothing was found in the cache so proceed with queuing the execution */ - try { - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - return queueInThread(); - } else { - return queueInSemaphore(); + final HystrixCommand _this = this; + + // create an Observable that will lazily execute when subscribed to + Observable o = Observable.create(new Func1, Subscription>() { + + @Override + public Subscription call(Observer observer) { + try { + /* used to track userThreadExecutionTime */ + invocationStartTime = System.currentTimeMillis(); + + // mark that we're starting execution on the ExecutionHook + executionHook.onStart(_this); + + /* determine if we're allowed to execute */ + if (!circuitBreaker.allowRequest()) { + // record that we are returning a short-circuited fallback + metrics.markShortCircuited(); + // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) + try { + observer.onNext(getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, "short-circuited")); + observer.onCompleted(); + } catch (Exception e) { + observer.onError(e); + } + return Subscriptions.empty(); + } else { + /* not short-circuited so proceed with queuing the execution */ + try { + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + return subscribeWithThreadIsolation(observer); + } else { + return subscribeWithSemaphoreIsolation(observer); + } + } catch (RuntimeException e) { + observer.onError(e); + return Subscriptions.empty(); + } + } + } finally { + recordExecutedCommand(); } - } catch (RuntimeException e) { + } + }); + + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + // wrap for timeout support + o = new TimeoutObservable(o, _this, performAsyncTimeout); + } + + // error handling + o = o.onErrorResumeNext(new Func1>() { + + @Override + public Observable call(Exception e) { // count that we are throwing an exception and re-throw it metrics.markExceptionThrown(); - throw e; + return Observable.error(e); } - } finally { - recordExecutedCommand(); + }); + + // we want to hand off work to a different scheduler so we don't tie up the Hystrix thread + o = o.observeOn(observeOn); + + o = o.finallyDo(new Action0() { + + @Override + public void call() { + Reference tl = timeoutTimer.get(); + if (tl != null) { + tl.clear(); + } + } + + }); + + // put in cache + if (isRequestCachingEnabled()) { + // wrap it for caching + o = new CachedObservableOriginal(o.cache(), this); + Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + if (fromCache != null) { + // another thread beat us so we'll use the cached value instead + o = new CachedObservableResponse((CachedObservableOriginal) fromCache, this); + } + // we just created an ObservableCommand so we cast and return it + return (ObservableCommand) o; + } else { + // no request caching so a simple wrapper just to pass 'this' along with the Observable + return new ObservableCommand(o, this); } } - private Future queueInSemaphore() { - TryableSemaphore executionSemaphore = getExecutionSemaphore(); - // acquire a permit - if (executionSemaphore.tryAcquire()) { - final CountDownLatch executionCompleted = new CountDownLatch(1); - try { - /** - * we want to run it synchronously so wrap a Future interface around the synchronous call that doesn't do any threading - *

- * we do this so that client code can execute .queue() and act as if its multi-threaded even if we choose to run it synchronously - *

- * We create the Future *before* execution so we can cache it. This allows us to dedupe calls rather than executing them all concurrently only then to find out we could have had them - * cached. - *

- * Theoretically we could do this all completely synchronously but because of caching we could have multiple threads still hitting the code in the Future we create so we need to have a - * CountdownLatch to make the get() block until execution is completed. - */ + /** + * Wraps a source Observable and remembers the original HystrixCommand. + *

+ * Used for request caching so multiple commands can respond from a single Observable but also get access to the originating HystrixCommand. + * + * @param + */ + private static class CachedObservableOriginal extends ObservableCommand { + + final HystrixCommand originalCommand; + + CachedObservableOriginal(final Observable actual, HystrixCommand command) { + super(new Func1, Subscription>() { + + @Override + public Subscription call(final Observer observer) { + return actual.subscribe(observer); + } + }, command); + this.originalCommand = command; + } + } + + private static class ObservableCommand extends Observable { + private final HystrixCommand command; + + ObservableCommand(Func1, Subscription> func, final HystrixCommand command) { + super(func); + this.command = command; + } + + public HystrixCommand getCommand() { + return command; + } + + ObservableCommand(final Observable originalObservable, final HystrixCommand command) { + super(new Func1, Subscription>() { + + @Override + public Subscription call(Observer observer) { + return originalObservable.subscribe(observer); + } + }); + this.command = command; + } + + } + + /** + * Wraps a CachedObservableOriginal as it is being returned from cache. + *

+ * As the Observable completes it copies state used for ExecutionResults + * and metrics that differentiate between the original and the de-duped "response from cache" command execution. + * + * @param + */ + private static class CachedObservableResponse extends ObservableCommand { + final CachedObservableOriginal originalObservable; + + CachedObservableResponse(final CachedObservableOriginal originalObservable, final HystrixCommand commandOfDuplicateCall) { + super(new Func1, Subscription>() { + + @Override + public Subscription call(final Observer observer) { + return originalObservable.subscribe(new Observer() { + + @Override + public void onCompleted() { + completeCommand(); + observer.onCompleted(); + } + + @Override + public void onError(Exception e) { + completeCommand(); + observer.onError(e); + } + + @Override + public void onNext(R v) { + observer.onNext(v); + } + + private void completeCommand() { + // when the observable completes we then update the execution results of the duplicate command + // set this instance to the result that is from cache + commandOfDuplicateCall.executionResult = originalObservable.originalCommand.executionResult; + // add that this came from cache + commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.addEvents(HystrixEventType.RESPONSE_FROM_CACHE); + // set the execution time to 0 since we retrieved from cache + commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.setExecutionTime(-1); + // record that this command executed + commandOfDuplicateCall.recordExecutedCommand(); + } + }); + } + }, commandOfDuplicateCall); + this.originalObservable = originalObservable; + } + + /* + * This is a cached response so we want the command of the observable we're wrapping. + */ + public HystrixCommand getCommand() { + return originalObservable.originalCommand; + } + } + + private static class TimeoutObservable extends Observable { + + public TimeoutObservable(final Observable o, final HystrixCommand originalCommand, final boolean isNonBlocking) { + super(new Func1, Subscription>() { + + @Override + public Subscription call(final Observer observer) { + final AtomicObservableSubscription s = new AtomicObservableSubscription(); - final AtomicReference value = new AtomicReference(); - CommandFuture responseFuture = new CommandFuture() { + TimerListener listener = new TimerListener() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } + @Override + public void tick() { + if (originalCommand.isCommandTimedOut.compareAndSet(false, true)) { + // do fallback logic - @Override - public boolean isCancelled() { - return false; - } + // report timeout failure + originalCommand.metrics.markTimeout(System.currentTimeMillis() - originalCommand.invocationStartTime); - @Override - public boolean isDone() { - return true; - } + // we record execution time because we are returning before + originalCommand.recordTotalExecutionTime(originalCommand.invocationStartTime); - @Override - public R get() throws InterruptedException, ExecutionException { - // wait for the execution to complete - executionCompleted.await(); - return value.get(); - } + try { + R v = originalCommand.getFallbackOrThrowException(HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", new TimeoutException()); + observer.onNext(v); + observer.onCompleted(); + } catch (HystrixRuntimeException re) { + observer.onError(re); + } + } - @Override - public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return get(); - } + s.unsubscribe(); + } - @Override - public ExecutionResult getExecutionResult() { - return executionResult; - } - }; - - // put in cache before executing so if multiple threads all try and execute duplicate commands we can de-dupe it - // they will each receive the same Future and block on the executionCompleted CountDownLatch until the execution below on the first - // thread completes at which point all threads who receive this cached Future will unblock and receive the same result - if (isRequestCachingEnabled()) { - Future fromCache = requestCache.putIfAbsent(getCacheKey(), responseFuture); - if (fromCache != null) { - // another thread beat us so let's return it from the cache and skip executing the one we just created - /* mark that we received this response from cache */ - metrics.markResponseFromCache(); - return asCachedFuture(fromCache); + @Override + public int getIntervalTimeInMilliseconds() { + return originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); + } + }; + + Reference _tl = null; + if (isNonBlocking) { + /* + * Scheduling a separate timer to do timeouts is more expensive + * so we'll only do it if we're being used in a non-blocking manner. + */ + _tl = HystrixTimer.getInstance().addTimerListener(listener); + } else { + /* + * Otherwise we just set the hook that queue().get() can trigger if a timeout occurs. + * + * This allows the blocking and non-blocking approaches to be coded basically the same way + * though it is admittedly awkward if we were just blocking (the use of Reference annoys me for example) + */ + _tl = new SoftReference(listener); } + final Reference tl = _tl; + + // set externally so execute/queue can see this + originalCommand.timeoutTimer.set(tl); + + return s.wrap(o.subscribe(new Observer() { + + @Override + public void onCompleted() { + tl.clear(); + observer.onCompleted(); + } + + @Override + public void onError(Exception e) { + tl.clear(); + observer.onError(e); + } + + @Override + public void onNext(R v) { + observer.onNext(v); + } + + })); } + }); + } + } + private Subscription subscribeWithSemaphoreIsolation(final Observer observer) { + TryableSemaphore executionSemaphore = getExecutionSemaphore(); + // acquire a permit + if (executionSemaphore.tryAcquire()) { + try { try { // store the command that is being run Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); @@ -634,103 +1082,106 @@ public ExecutionResult getExecutionResult() { // execute outside of future so that fireAndForget will still work (ie. someone calls queue() but not get()) and so that multiple requests can be deduped through request caching R r = executeCommand(); r = executionHook.onComplete(this, r); - value.set(r); - - return responseFuture; + observer.onNext(r); + /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ + recordTotalExecutionTime(invocationStartTime); + /* now complete which releases the consumer */ + observer.onCompleted(); + // empty subscription since we executed synchronously + return Subscriptions.empty(); + } catch (Exception e) { + /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ + recordTotalExecutionTime(invocationStartTime); + observer.onError(e); + // empty subscription since we executed synchronously + return Subscriptions.empty(); } finally { // pop the command that is being run Hystrix.endCurrentThreadExecutingCommand(); } } finally { - // mark that we're completed - executionCompleted.countDown(); // release the semaphore executionSemaphore.release(); - - /* execution time on queue via semaphore */ - recordTotalExecutionTime(invocationStartTime.get()); } } else { metrics.markSemaphoreRejection(); logger.debug("HystrixCommand Execution Rejection by Semaphore."); // debug only since we're throwing the exception and someone higher will do something with it // retrieve a fallback or throw an exception if no fallback available - return asFuture(getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, "could not acquire a semaphore for execution")); + observer.onNext(getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, "could not acquire a semaphore for execution")); + observer.onCompleted(); + // empty subscription since we executed synchronously + return Subscriptions.empty(); } } - private Future queueInThread() { + private Subscription subscribeWithThreadIsolation(final Observer observer) { // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE) isExecutedInThread.set(true); // final reference to the current calling thread so the child thread can access it if needed final Thread callingThread = Thread.currentThread(); - // a snapshot of time so the thread can measure how long it waited before executing - final long timeBeforeExecution = System.currentTimeMillis(); - final HystrixCommand _this = this; - // wrap the synchronous execute() method in a Callable and execute in the threadpool - QueuedExecutionFuture future = new QueuedExecutionFuture(this, threadPool.getExecutor(), new HystrixContextCallable(new Callable() { + try { + if (!threadPool.isQueueSpaceAvailable()) { + // we are at the property defined max so want to throw a RejectedExecutionException to simulate reaching the real max + throw new RejectedExecutionException("Rejected command because thread-pool queueSize is at rejection threshold."); + } + + // wrap the synchronous execute() method in a Callable and execute in the threadpool + final Future f = threadPool.getExecutor().submit(concurrencyStrategy.wrapCallable(new HystrixContextCallable(new Callable() { - @Override - public R call() throws Exception { - try { - // assign 'callingThread' to our NFExceptionThreadingUtility ThreadLocal variable so that if we blow up - // anywhere along the way the exception knows who the calling thread is and can include it in the stacktrace - ExceptionThreadingUtility.assignCallingThread(callingThread); + @Override + public R call() throws Exception { + try { + // assign 'callingThread' to our NFExceptionThreadingUtility ThreadLocal variable so that if we blow up + // anywhere along the way the exception knows who the calling thread is and can include it in the stacktrace + ExceptionThreadingUtility.assignCallingThread(callingThread); - // execution hook - executionHook.onThreadStart(_this); + // execution hook + executionHook.onThreadStart(_this); - // count the active thread - threadPool.markThreadExecution(); + // count the active thread + threadPool.markThreadExecution(); - // see if this command should still be executed, or if the requesting thread has walked away (timed-out) already - long timeQueued = System.currentTimeMillis() - timeBeforeExecution; - if (isCommandTimedOut.get() || timeQueued > properties.executionIsolationThreadTimeoutInMilliseconds().get()) { - /* - * We check isCommandTimedOut first as that is what most time outs will result in. - * We also check the actual time because fireAndForget() will never result in isCommandTimedOut=true since the user-thread never calls the Future.get() method. - * Thus, we want to ensure we don't continue with execution below if we're past the timeout duration regardless of whether the Future.get() was invoked (such as - * fireAndForget or when the user kicks of many - * calls asynchronously to come back to them later) - */ - if (logger.isDebugEnabled()) { - logger.debug("Callable is being skipped since the user-thread has already timed-out this request after " + timeQueued + "ms."); - } - if (isCommandTimedOut.get()) { - // we don't need to mark any stats here as that will have already been done in the Future.get() method - } else { - // try setting it if the Future.get() hasn't already done it - if (isCommandTimedOut.compareAndSet(false, true)) { - // the Future.get() method has not been called so we'll mark it here (this can happen on fireAndForget executions) - metrics.markTimeout(timeQueued); + try { + // store the command that is being run + Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); + + // execute the command + R r = executeCommand(); + if (isCommandTimedOut.get()) { + // state changes before termination + preTerminationWork(); + return null; } + // give the hook an opportunity to modify it + r = executionHook.onComplete(_this, r); + // pass to the observer + observer.onNext(r); + // state changes before termination + preTerminationWork(); + /* now complete which releases the consumer */ + observer.onCompleted(); + return r; + } finally { + // pop this off the thread now that it's done + Hystrix.endCurrentThreadExecutingCommand(); } - - return null; + } catch (Exception e) { + // state changes before termination + preTerminationWork(); + observer.onError(e); + throw e; } + } + + private void preTerminationWork() { + /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ + recordTotalExecutionTime(invocationStartTime); - try { - // store the command that is being run - Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); - - // execute the command - R r = executeCommand(); - return executionHook.onComplete(_this, r); - } finally { - // pop this off the thread now that it's done - Hystrix.endCurrentThreadExecutingCommand(); - } - } catch (Exception e) { - if (!isCommandTimedOut.get()) { - // count (if we didn't timeout) that we are throwing an exception and re-throw it - metrics.markExceptionThrown(); - } - throw e; - } finally { threadPool.markThreadCompletion(); try { @@ -739,27 +1190,32 @@ public R call() throws Exception { logger.warn("ExecutionHook.onThreadComplete threw an exception that will be ignored.", e); } } - } - })); - // put in cache BEFORE starting so we're sure that one-and-only-one Future exists - if (isRequestCachingEnabled()) { - /* - * NOTE: As soon as this Future is added another thread could retrieve it and call get() before we return from this method. - */ - Future fromCache = requestCache.putIfAbsent(getCacheKey(), future); - if (fromCache != null) { - // another thread beat us so let's return it from the cache and skip executing the one we just created - /* mark that we received this response from cache */ - metrics.markResponseFromCache(); - return asCachedFuture(fromCache); - } - } + }))); + + return new Subscription() { + + @Override + public void unsubscribe() { + f.cancel(properties.executionIsolationThreadInterruptOnTimeout().get()); + } - // start execution and throw an exception if rejection occurs - future.start(true); + }; - return future; + } catch (RejectedExecutionException e) { + // mark on counter + metrics.markThreadPoolRejection(); + // use a fallback instead (or throw exception if not implemented) + observer.onNext(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", e)); + observer.onCompleted(); + return Subscriptions.empty(); + } catch (Exception e) { + // unknown exception + logger.error(getLogMessagePrefix() + ": Unexpected exception while submitting to queue.", e); + observer.onNext(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "had unexpected exception while attempting to queue for execution.", e)); + observer.onCompleted(); + return Subscriptions.empty(); + } } /** @@ -828,10 +1284,10 @@ private R executeCommand() { // this means we have already timed out then we don't count this error stat and we just return // as this means the user-thread has already returned, we've already done fallback logic // and we've already counted the timeout stat - logger.error("Error executing HystrixCommand.run() [TimedOut]. Proceeding to fallback logic ...", e); + logger.debug("Error executing HystrixCommand.run() [TimedOut]. Proceeding to fallback logic ...", e); return null; } else { - logger.error("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", e); + logger.debug("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", e); } // report failure metrics.markFailure(System.currentTimeMillis() - startTime); @@ -1144,7 +1600,7 @@ private R getFallbackOrThrowException(HystrixEventType eventType, FailureType fa throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and no fallback available.", e, fe); } catch (Exception fe) { - logger.error("Error retrieving fallback for HystrixCommand. ", fe); + logger.debug("HystrixCommand execution " + failureType.name() + " and fallback retrieval failed.", fe); metrics.markFallbackFailure(); // record the executionResult executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_FAILURE); @@ -1197,470 +1653,73 @@ private R getFallbackOrThrowException(HystrixEventType eventType, FailureType fa * This being immutable forces and ensure thread-safety instead of using AtomicInteger/ConcurrentLinkedQueue and determining * when it's safe to mutate the object directly versus needing to deep-copy clone to a new instance. */ - private static class ExecutionResult { - private final List events; - private final int executionTime; - private final Exception exception; - - private ExecutionResult(HystrixEventType... events) { - this(Arrays.asList(events), -1, null); - } - - public ExecutionResult setExecutionTime(int executionTime) { - return new ExecutionResult(events, executionTime, exception); - } - - public ExecutionResult setException(Exception e) { - return new ExecutionResult(events, executionTime, e); - } - - private ExecutionResult(List events, int executionTime, Exception e) { - // we are safe assigning the List reference instead of deep-copying - // because we control the original list in 'newEvent' - this.events = events; - this.executionTime = executionTime; - this.exception = e; - } - - // we can return a static version since it's immutable - private static ExecutionResult EMPTY = new ExecutionResult(new HystrixEventType[0]); - - /** - * Creates a new ExecutionResult by adding the defined 'events' to the ones on the current instance. - * - * @param events - * @return - */ - public ExecutionResult addEvents(HystrixEventType... events) { - // TODO are there performance reasons for this be changed to a persistent or concurrent data structure so we can append without copying? - ArrayList newEvents = new ArrayList(); - newEvents.addAll(this.events); - for (HystrixEventType e : events) { - newEvents.add(e); - } - return new ExecutionResult(Collections.unmodifiableList(newEvents), executionTime, exception); - } - } - - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* RequestCache */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * Key to be used for request caching. - *

- * By default this returns null which means "do not cache". - *

- * To enable caching override this method and return a string key uniquely representing the state of a command instance. - *

- * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. - * - * @return cacheKey - */ - protected String getCacheKey() { - return null; - } - - private Future asFutureForCache(final R value) { - return asFuture(value); - } - - /** - * This wrapper is used for handling Future responses when doing request caching. - *

- * When a response is returned we need to copy the state from the original HystrixCommand instance that actually executed the Future to the current HystrixCommand instance which is - * returning from cache. - *

- * The state is contained within the ExecutionResult object and tells what happened during execution and includes an exception for re-throwing if applicable. - * - * @param - * - */ - private Future asCachedFuture(Future actualFuture) { - - if (!(actualFuture instanceof CommandFuture)) { - throw new RuntimeException("This should be a CommandFuture from the asFutureForCache method."); - } - - final CommandFuture commandFuture = (CommandFuture) actualFuture; - - return new Future() { - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return commandFuture.cancel(mayInterruptIfRunning); - } - - @Override - public boolean isCancelled() { - return commandFuture.isCancelled(); - } - - @Override - public boolean isDone() { - return commandFuture.isDone(); - } - - @Override - public R get() throws InterruptedException, ExecutionException { - try { - return commandFuture.get(); - } finally { - // set this instance to the result that is from cache - executionResult = commandFuture.getExecutionResult(); - // add that this came from cache - executionResult = executionResult.addEvents(HystrixEventType.RESPONSE_FROM_CACHE); - // set the execution time to 0 since we retrieved from cache - executionResult = executionResult.setExecutionTime(-1); - } - } - - @Override - public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - try { - return commandFuture.get(timeout, unit); - } finally { - // set this instance to the result that is from cache - executionResult = commandFuture.getExecutionResult(); - // add that this came from cache - executionResult = executionResult.addEvents(HystrixEventType.RESPONSE_FROM_CACHE); - // set the execution time to 0 since we retrieved from cache - executionResult = executionResult.setExecutionTime(-1); - } - } - - }; - } - - private boolean isRequestCachingEnabled() { - return properties.requestCacheEnabled().get(); - } - - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* Wrapper around Future so we can control timeouts */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * This wrapper around Future allows extending the get method to always include timeout functionality. - *

- * We do not want developers queueing up commands and calling the normal get() and blocking indefinitely. - *

- * This implementation routes all get() calls to get(long timeout, TimeUnit unit) so that timeouts occur automatically for commands executed via execute() or - * queue().get() - */ - private class QueuedExecutionFuture implements CommandFuture { - private final ThreadPoolExecutor executor; - private final Callable callable; - private final CountDownLatch actualResponseReceived = new CountDownLatch(1); - private final AtomicBoolean actualFutureExecuted = new AtomicBoolean(false); - private volatile R result; // the result of the get() - private volatile ExecutionException executionException; // in case an exception is thrown - private volatile HystrixRuntimeException rejectedException; - private volatile Future actualFuture = null; - private volatile boolean isInterrupted = false; - private final CountDownLatch futureStarted = new CountDownLatch(1); - private final AtomicBoolean started = new AtomicBoolean(false); - - public QueuedExecutionFuture(HystrixCommand command, ThreadPoolExecutor executor, Callable callable) { - this.executor = executor; - this.callable = callable; - } - - private void start() { - start(false); - } - - /** - * Start execution of Callable on ThreadPoolExecutor - * - * @param throwIfRejected - * since we want an exception thrown in the main queue() path but not via cached responses - */ - private void start(boolean throwIfRejected) { - // make sure we only start once - if (started.compareAndSet(false, true)) { - try { - if (!threadPool.isQueueSpaceAvailable()) { - // we are at the property defined max so want to throw a RejectedExecutionException to simulate reaching the real max - throw new RejectedExecutionException("Rejected command because thread-pool queueSize is at rejection threshold."); - } - // allow the ConcurrencyStrategy to wrap the Callable if desired and then submit to the ThreadPoolExecutor - actualFuture = executor.submit(concurrencyStrategy.wrapCallable(callable)); - } catch (RejectedExecutionException e) { - // mark on counter - metrics.markThreadPoolRejection(); - // use a fallback instead (or throw exception if not implemented) - try { - actualFuture = asFuture(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", e)); - } catch (HystrixRuntimeException hre) { - actualFuture = asFuture(hre); - // store this so it can be thrown to queue() - rejectedException = hre; - } - } catch (Exception e) { - // unknown exception - logger.error(getLogMessagePrefix() + ": Unexpected exception while submitting to queue.", e); - try { - actualFuture = asFuture(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "had unexpected exception while attempting to queue for execution.", e)); - } catch (HystrixRuntimeException hre) { - actualFuture = asFuture(hre); - throw hre; - } - } finally { - futureStarted.countDown(); - } - } else { - /* - * This else path can occur when: - * - * - HystrixCommand.getCacheKey() is implemented - * - multiple threads execute a command concurrently with the same cache key - * - each command returns the same Future - * - as the Future is submitted we want only 1 thread to submit in the if block above - * - other threads waiting on the same Future will come through this else path and wait - */ - try { - // wait for if block above to finish on a different thread - futureStarted.await(); - } catch (InterruptedException e) { - isInterrupted = true; - logger.error(getLogMessagePrefix() + ": Unexpected interruption while waiting on other thread submitting to queue.", e); - actualFuture = asFuture(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "Unexpected interruption while waiting on other thread submitting to queue.", e)); - } - } - - if (throwIfRejected && rejectedException != null) { - throw rejectedException; - } - } - - /** - * We override the get(long timeout, TimeUnit unit) to handle timeouts, fallbacks, etc. - */ - @Override - public R get(long timeout, TimeUnit unit) throws CancellationException, InterruptedException, ExecutionException { - /* in case another thread got to this (via cache) before the constructing thread started it, we'll optimistically try to start it and start() will ensure only one time wins */ - start(); - // now we try to get the response - if (actualFutureExecuted.compareAndSet(false, true)) { - // 1 thread will execute this 1 time - performActualGet(); - } else { - // all other threads/requests will go here waiting on performActualGet completing - actualResponseReceived.await(); - /** - * I am considering putting a timeout value in this latch.await() even though performActualGet seems - * like it should never NOT properly call latch.countDown(). - * - * One scenario I'm trying to determine if it can ever happen is a thread interrupt that prevents the finally block - * from doing the latch.countDown(). - * - * http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html - * - * Note: If the JVM exits while the try or catch code is being executed, then the finally block may not execute. - * Likewise, if the thread executing the try or catch code is interrupted or killed, the finally block may not - * execute even though the application as a whole continues. - */ - } - - if (executionException != null) { - throw executionException; - } else { - return result; - } - } - - /** - * We override the get() to force it to always have a timeout so developers can not - * accidentally use the HystrixCommand.queue().get() methods and block indefinitely. - */ - @Override - public R get() throws CancellationException, InterruptedException, ExecutionException { - return get(properties.executionIsolationThreadTimeoutInMilliseconds().get(), TimeUnit.MILLISECONDS); - } - - /** - * The actual Future.get() that we want only 1 thread to perform once. - *

- * NOTE: This sets mutable variables on this Future instance so as to do so in a thread-safe manner. - *

- * The execution of this method is protected by a CountDownLatch to occur only once and by a single thread. - *

- * As soon as this method returns all other threads are released so the correct state must be set before this method returns. - * - * @return - * @throws CancellationException - * @throws InterruptedException - * @throws ExecutionException - */ - private void performActualGet() throws CancellationException, InterruptedException, ExecutionException { - try { - // this check needs to be inside the try/finally so even if an exception is thrown - // we will countDown the latch and release threads - if (!started.get() || actualFuture == null) { - /** - * https://github.com/Netflix/Hystrix/issues/113 - * - * Output any extra information that can help tracking down how this failed - * as it most likely means there's a concurrency bug. - */ - throw new IllegalStateException("Response Not Available. Key: " - + getCommandKey().name() + " ActualFuture: " + actualFuture - + " Started: " + started.get() + " actualFutureExecuted: " + actualFutureExecuted.get() - + " futureStarted: " + futureStarted.getCount() - + " isInterrupted: " + isInterrupted - + " actualResponseReceived: " + actualResponseReceived.getCount() - + " isCommandTimedOut: " + isCommandTimedOut.get() - + " Events: " + Arrays.toString(getExecutionEvents().toArray())); - } - // get on the actualFuture with timeout values from properties - result = actualFuture.get(properties.executionIsolationThreadTimeoutInMilliseconds().get(), TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - // try to cancel the future (interrupt it) - actualFuture.cancel(properties.executionIsolationThreadInterruptOnTimeout().get()); - // mark this command as timed-out so the run() when it completes can ignore it - if (isCommandTimedOut.compareAndSet(false, true)) { - // report timeout failure (or skip this if the compareAndSet failed as that means a thread-race occurred with the execution as the object lived in the queue too long) - metrics.markTimeout(System.currentTimeMillis() - invocationStartTime.get()); - } - - try { - result = getFallbackOrThrowException(HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", e); - } catch (HystrixRuntimeException re) { - // we want to obey the contract of NFFuture.get() and throw an ExecutionException rather than a random RuntimeException that developers wouldn't expect - executionException = new ExecutionException(re); - // we can't capture this in execute/queue so we do it here - metrics.markExceptionThrown(); - } - } catch (ExecutionException e) { - // if the actualFuture itself throws an ExcecutionException we want to capture it - executionException = e; - } finally { - // mark that we are done and other threads can proceed - actualResponseReceived.countDown(); - - /* execution time on threaded execution */ - recordTotalExecutionTime(invocationStartTime.get()); - } - } + private static class ExecutionResult { + private final List events; + private final int executionTime; + private final Exception exception; - /** - * Allow retrieving the executionResult from 1 Future in another Future (due to request caching). - * - * @return - */ - public ExecutionResult getExecutionResult() { - return executionResult; + private ExecutionResult(HystrixEventType... events) { + this(Arrays.asList(events), -1, null); } - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - // we don't want to allow canceling - return false; + public ExecutionResult setExecutionTime(int executionTime) { + return new ExecutionResult(events, executionTime, exception); } - @Override - public boolean isCancelled() { - /* in case another thread got to this (via cache) before the constructing thread started it, we'll optimistically try to start it and start() will ensure only one time wins */ - start(); - /* now 'actualFuture' will be set to something */ - return actualFuture.isCancelled(); + public ExecutionResult setException(Exception e) { + return new ExecutionResult(events, executionTime, e); } - @Override - public boolean isDone() { - /* in case another thread got to this (via cache) before the constructing thread started it, we'll optimistically try to start it and start() will ensure only one time wins */ - start(); - /* now 'actualFuture' will be set to something */ - return actualFuture.isDone(); + private ExecutionResult(List events, int executionTime, Exception e) { + // we are safe assigning the List reference instead of deep-copying + // because we control the original list in 'newEvent' + this.events = events; + this.executionTime = executionTime; + this.exception = e; } - } + // we can return a static version since it's immutable + private static ExecutionResult EMPTY = new ExecutionResult(new HystrixEventType[0]); - private static interface CommandFuture extends Future { /** - * Allow retrieving the executionResult from 1 Future in another Future (due to request caching). + * Creates a new ExecutionResult by adding the defined 'events' to the ones on the current instance. * - * @return ExecutionResult + * @param events + * @return */ - public ExecutionResult getExecutionResult(); - - } - - private Future asFuture(final R value) { - return new CommandFuture() { - - @Override - public boolean cancel(boolean arg0) { - return false; - } - - @Override - public R get() throws InterruptedException, ExecutionException { - return value; - } - - @Override - public R get(long arg0, TimeUnit arg1) throws InterruptedException, ExecutionException, TimeoutException { - return get(); - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public ExecutionResult getExecutionResult() { - return executionResult; + public ExecutionResult addEvents(HystrixEventType... events) { + ArrayList newEvents = new ArrayList(); + newEvents.addAll(this.events); + for (HystrixEventType e : events) { + newEvents.add(e); } - - }; + return new ExecutionResult(Collections.unmodifiableList(newEvents), executionTime, exception); + } } - private Future asFuture(final HystrixRuntimeException e) { - return new CommandFuture() { - - @Override - public boolean cancel(boolean arg0) { - return false; - } - - @Override - public R get() throws InterruptedException, ExecutionException { - throw new ExecutionException(e); - } - - @Override - public R get(long arg0, TimeUnit arg1) throws InterruptedException, ExecutionException, TimeoutException { - return get(); - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* RequestCache */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ - @Override - public ExecutionResult getExecutionResult() { - return executionResult; - } + /** + * Key to be used for request caching. + *

+ * By default this returns null which means "do not cache". + *

+ * To enable caching override this method and return a string key uniquely representing the state of a command instance. + *

+ * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. + * + * @return cacheKey + */ + protected String getCacheKey() { + return null; + } - }; + private boolean isRequestCachingEnabled() { + return properties.requestCacheEnabled().get(); } /* ******************************************************************************** */ @@ -1928,7 +1987,7 @@ public void testExecutionMultipleTimes() { // second should fail command.execute(); fail("we should not allow this ... it breaks the state of request logs"); - } catch (Exception e) { + } catch (IllegalStateException e) { e.printStackTrace(); // we want to get here } @@ -1937,7 +1996,7 @@ public void testExecutionMultipleTimes() { // queue should also fail command.queue(); fail("we should not allow this ... it breaks the state of request logs"); - } catch (Exception e) { + } catch (IllegalStateException e) { e.printStackTrace(); // we want to get here } @@ -2240,48 +2299,280 @@ public void testQueueFailureWithFallback() { assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution (asynchronously) that fails, has getFallback implemented but that fails as well. + */ + @Test + public void testQueueFailureWithFallbackFailure() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + e.printStackTrace(); + assertNotNull(de.getFallbackException()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveSuccess() { + try { + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.observe().toBlockingObservable().single()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(null, command.getFailedExecutionException()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnScheduler() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + Scheduler customScheduler = new Scheduler() { + + private final Scheduler self = this; + + @Override + public Subscription schedule(T state, Func2 action) { + return schedule(state, action, 0, TimeUnit.MILLISECONDS); + } + + @Override + public Subscription schedule(final T state, final Func2 action, long delayTime, TimeUnit unit) { + new Thread("RxScheduledThread") { + @Override + public void run() { + action.call(self, state); + } + }.start(); + + // not testing unsubscribe behavior + return Subscriptions.empty(); + } + + }; + + command.toObservable(customScheduler).subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Exception e) { + latch.countDown(); + e.printStackTrace(); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + assertTrue(commandThread.get().getName().startsWith("hystrix-")); + assertTrue(subscribeThread.get().getName().equals("RxScheduledThread")); + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnComputationSchedulerByDefaultForThreadIsolation() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Exception e) { + latch.countDown(); + e.printStackTrace(); + + } - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + assertTrue(commandThread.get().getName().startsWith("hystrix-")); + assertTrue(subscribeThread.get().getName().startsWith("RxComputationThreadPool")); } /** - * Test a command execution (asynchronously) that fails, has getFallback implemented but that fails as well. + * Test a successful command execution. */ @Test - public void testQueueFailureWithFallbackFailure() { - TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - if (e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + public void testObserveOnImmediateSchedulerByDefaultForSemaphoreIsolation() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Exception e) { + latch.countDown(); e.printStackTrace(); - assertNotNull(de.getFallbackException()); - } else { - fail("the cause should be HystrixRuntimeException"); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); } - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + String mainThreadName = Thread.currentThread().getName(); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + // semaphore should be on the calling thread + assertTrue(commandThread.get().getName().equals(mainThreadName)); + assertTrue(subscribeThread.get().getName().equals(mainThreadName)); } /** @@ -2910,6 +3201,127 @@ public void testQueuedExecutionTimeoutFallbackFailure() { assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); } + /** + * Test a queued command execution timeout where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.observe().toBlockingObservable().single()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + /** * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. */ @@ -4252,6 +4664,71 @@ public void testNoRequestCacheViaExecuteSemaphore1() { assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); } + @Test + public void testNoRequestCacheOnTimeoutThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + NoRequestCacheTimeoutWithoutFallback r1 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + System.out.println("r1 value: " + r1.execute()); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r2 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r2.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r2.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r3 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + Future f3 = r3.queue(); + try { + f3.get(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (ExecutionException e) { + e.printStackTrace(); + assertTrue(r3.isResponseTimedOut()); + // what we want + } + + Thread.sleep(500); // timeout on command is set to 200ms + + NoRequestCacheTimeoutWithoutFallback r4 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r4.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r4.isResponseTimedOut()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + @Test public void testRequestCacheOnTimeoutCausesNullPointerException() throws Exception { TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); @@ -4313,7 +4790,7 @@ public void testRequestCacheOnTimeoutThrowsException() throws Exception { TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); RequestCacheTimeoutWithoutFallback r1 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); try { - r1.execute(); + System.out.println("r1 value: " + r1.execute()); // we should have thrown an exception fail("expected a timeout"); } catch (HystrixRuntimeException e) { @@ -4399,18 +4876,17 @@ public void testRequestCacheOnThreadRejectionThrowsException() throws Exception } RequestCacheThreadRejectionWithoutFallback r3 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); - Future f3 = r3.queue(); try { - System.out.println("f3: " + f3.get()); + System.out.println("f3: " + r3.queue().get()); // we should have thrown an exception fail("expected a rejection"); - } catch (ExecutionException e) { + } catch (HystrixRuntimeException e) { // e.printStackTrace(); assertTrue(r3.isResponseRejected()); // what we want } - // let the command finish (only 1 should actually be blocked on this do to the response cache) + // let the command finish (only 1 should actually be blocked on this due to the response cache) completionLatch.countDown(); // then another after the command has completed @@ -5970,6 +6446,31 @@ protected Boolean getFallback() { } } + private static class NoRequestCacheTimeoutWithoutFallback extends TestHystrixCommand { + public NoRequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); + + // we want it to timeout + } + + @Override + protected Boolean run() { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); + } + return true; + } + + @Override + public String getCacheKey() { + return null; + } + } + /** * The run() will take time. No fallback implementation. */ @@ -6120,7 +6621,8 @@ protected Boolean run() { try { Thread.sleep(500); } catch (InterruptedException e) { - e.printStackTrace(); + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); } return true; } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java index a5b2d099b..b4fb1637d 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java @@ -17,6 +17,10 @@ import java.util.concurrent.Future; +import rx.Observable; +import rx.concurrency.Schedulers; + +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; import com.netflix.hystrix.exception.HystrixBadRequestException; import com.netflix.hystrix.exception.HystrixRuntimeException; @@ -56,4 +60,35 @@ public interface HystrixExecutable { */ public Future queue(); + /** + * Used for asynchronous execution of command with a callback by subscribing to the {@link Observable}. + *

+ * This eagerly starts execution of the command the same as {@link #queue()} and {@link #execute()}. + * A lazy {@link Observable} can be obtained from {@link HystrixCommand#toObservable()} or {@link HystrixCollapser#toObservable()}. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ * Use {@link HystrixCommand#toObservable(rx.Scheduler)} or {@link HystrixCollapser#toObservable(rx.Scheduler)} to schedule the callback differently. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable observe(); + } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java index 91ca2a71f..44de8c208 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java @@ -18,15 +18,17 @@ import static org.junit.Assert.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import rx.Observable; +import rx.Observer; +import rx.Subscription; +import rx.subscriptions.Subscriptions; +import rx.util.functions.Func1; + import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; @@ -54,15 +56,15 @@ public class HystrixRequestCache { *

* Key => CommandPrefix + CacheKey : Future from queue() */ - private static final HystrixRequestVariableHolder>> requestVariableForCache = new HystrixRequestVariableHolder>>(new HystrixRequestVariableLifecycle>>() { + private static final HystrixRequestVariableHolder>> requestVariableForCache = new HystrixRequestVariableHolder>>(new HystrixRequestVariableLifecycle>>() { @Override - public ConcurrentHashMap> initialValue() { - return new ConcurrentHashMap>(); + public ConcurrentHashMap> initialValue() { + return new ConcurrentHashMap>(); } @Override - public void shutdown(ConcurrentHashMap> value) { + public void shutdown(ConcurrentHashMap> value) { // nothing to shutdown }; @@ -104,11 +106,11 @@ private static HystrixRequestCache getInstance(RequestCacheKey rcKey, HystrixCon */ // suppressing warnings because we are using a raw Future since it's in a heterogeneous ConcurrentHashMap cache @SuppressWarnings({ "unchecked" }) - public Future get(String cacheKey) { + /* package */ Observable get(String cacheKey) { ValueCacheKey key = getRequestCacheKey(cacheKey); if (key != null) { /* look for the stored value */ - return (Future) requestVariableForCache.get(concurrencyStrategy).get(key); + return (Observable) requestVariableForCache.get(concurrencyStrategy).get(key); } return null; } @@ -127,11 +129,11 @@ public Future get(String cacheKey) { */ // suppressing warnings because we are using a raw Future since it's in a heterogeneous ConcurrentHashMap cache @SuppressWarnings({ "unchecked" }) - public Future putIfAbsent(String cacheKey, Future f) { + /* package */ Observable putIfAbsent(String cacheKey, Observable f) { ValueCacheKey key = getRequestCacheKey(cacheKey); if (key != null) { /* look for the stored value */ - Future alreadySet = (Future) requestVariableForCache.get(concurrencyStrategy).putIfAbsent(key, f); + Observable alreadySet = (Observable) requestVariableForCache.get(concurrencyStrategy).putIfAbsent(key, f); if (alreadySet != null) { // someone beat us so we didn't cache this return alreadySet; @@ -282,17 +284,17 @@ public void testCache() { HystrixRequestContext context = HystrixRequestContext.initializeContext(); try { HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); - cache1.putIfAbsent("valueA", new TestFuture("a1")); - cache1.putIfAbsent("valueA", new TestFuture("a2")); - cache1.putIfAbsent("valueB", new TestFuture("b1")); + cache1.putIfAbsent("valueA", new TestObservable("a1")); + cache1.putIfAbsent("valueA", new TestObservable("a2")); + cache1.putIfAbsent("valueB", new TestObservable("b1")); HystrixRequestCache cache2 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command2"), strategy); - cache2.putIfAbsent("valueA", new TestFuture("a3")); + cache2.putIfAbsent("valueA", new TestObservable("a3")); - assertEquals("a1", cache1.get("valueA").get()); - assertEquals("b1", cache1.get("valueB").get()); + assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); + assertEquals("b1", cache1.get("valueB").toBlockingObservable().last()); - assertEquals("a3", cache2.get("valueA").get()); + assertEquals("a3", cache2.get("valueA").toBlockingObservable().last()); assertNull(cache2.get("valueB")); } catch (Exception e) { fail("Exception: " + e.getMessage()); @@ -318,8 +320,8 @@ public void testClearCache() { HystrixRequestContext context = HystrixRequestContext.initializeContext(); try { HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); - cache1.putIfAbsent("valueA", new TestFuture("a1")); - assertEquals("a1", cache1.get("valueA").get()); + cache1.putIfAbsent("valueA", new TestObservable("a1")); + assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); cache1.clear("valueA"); assertNull(cache1.get("valueA")); } catch (Exception e) { @@ -330,39 +332,19 @@ public void testClearCache() { } } - private static class TestFuture implements Future { - - private final String value; - - public TestFuture(String value) { - this.value = value; - } + private static class TestObservable extends Observable { + public TestObservable(final String value) { + super(new Func1, Subscription>() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } + @Override + public Subscription call(Observer observer) { + observer.onNext(value); + observer.onCompleted(); + return Subscriptions.empty(); + } - @Override - public boolean isCancelled() { - return false; + }); } - - @Override - public boolean isDone() { - return false; - } - - @Override - public String get() throws InterruptedException, ExecutionException { - return value; - } - - @Override - public String get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return value; - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java new file mode 100644 index 000000000..9f655748f --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java @@ -0,0 +1,354 @@ +package com.netflix.hystrix.collapser; + +import static org.junit.Assert.*; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Test; + +import rx.Observable; +import rx.Observer; +import rx.Subscription; +import rx.subscriptions.BooleanSubscription; +import rx.util.functions.Func1; + +import com.netflix.hystrix.HystrixCollapser.CollapsedRequest; + +/** + * The Observable that represents a collapsed request sent back to a user. + *

+ * This is an internal implementation class that combines the Observable and CollapsedRequest functionality. + *

+ * We publicly expose these via interfaces only since we want clients to only see Observable and implementors to only see CollapsedRequest, not the combination of the two. + * + * @param + * + * @param + */ +/* package */class CollapsedRequestObservableFunction implements CollapsedRequest, Func1, Subscription> { + private final R argument; + private final AtomicReference> rh = new AtomicReference>(new CollapsedRequestObservableFunction.ResponseHolder()); + private final BooleanSubscription subscription = new BooleanSubscription(); + + public CollapsedRequestObservableFunction(R arg) { + this.argument = arg; + } + + /** + * The request argument. + * + * @return request argument + */ + @Override + public R getArgument() { + return argument; + } + + /** + * When set any client thread blocking on get() will immediately be unblocked and receive the response. + * + * @throws IllegalStateException + * if called more than once or after setException. + * @param response + */ + @Override + public void setResponse(T response) { + while (true) { + if (subscription.isUnsubscribed()) { + return; + } + CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); + if (r.getResponse() != null) { + throw new IllegalStateException("setResponse can only be called once"); + } + if (r.getException() != null) { + throw new IllegalStateException("Exception is already set so response can not be => Response: " + response + " subscription: " + subscription.isUnsubscribed() + " observer: " + r.getObserver() + " Exception: " + r.getException().getMessage(), r.getException()); + } + + CollapsedRequestObservableFunction.ResponseHolder nr = new CollapsedRequestObservableFunction.ResponseHolder(r.getObserver(), response, r.getException()); + if (rh.compareAndSet(r, nr)) { + // success + sendResponseIfRequired(subscription, nr); + break; + } else { + // we'll retry + } + } + } + + /** + * Set an exception if a response is not yet received otherwise skip it + * + * @param e + */ + public void setExceptionIfResponseNotReceived(Exception e) { + while (true) { + if (subscription.isUnsubscribed()) { + return; + } + CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); + // only proceed if neither response is set + if (r.getResponse() == null && r.getException() == null) { + CollapsedRequestObservableFunction.ResponseHolder nr = new CollapsedRequestObservableFunction.ResponseHolder(r.getObserver(), r.getResponse(), e); + if (rh.compareAndSet(r, nr)) { + // success + sendResponseIfRequired(subscription, nr); + break; + } else { + // we'll retry + } + } else { + // return quietly instead of throwing an exception + break; + } + } + } + + /** + * When set any client thread blocking on get() will immediately be unblocked and receive the exception. + * + * @throws IllegalStateException + * if called more than once or after setResponse. + * @param response + */ + @Override + public void setException(Exception e) { + while (true) { + if (subscription.isUnsubscribed()) { + return; + } + CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); + if (r.getException() != null) { + throw new IllegalStateException("setException can only be called once"); + } + if (r.getResponse() != null) { + throw new IllegalStateException("Response is already set so exception can not be => Response: " + r.getResponse() + " Exception: " + e.getMessage(), e); + } + + CollapsedRequestObservableFunction.ResponseHolder nr = new CollapsedRequestObservableFunction.ResponseHolder(r.getObserver(), r.getResponse(), e); + if (rh.compareAndSet(r, nr)) { + // success + sendResponseIfRequired(subscription, nr); + break; + } else { + // we'll retry + } + } + } + + @Override + public Subscription call(Observer observer) { + while (true) { + CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); + if (r.getObserver() != null) { + throw new IllegalStateException("Only 1 Observer can subscribe. Use multicast/publish/cache/etc for multiple subscribers."); + } + CollapsedRequestObservableFunction.ResponseHolder nr = new CollapsedRequestObservableFunction.ResponseHolder(observer, r.getResponse(), r.getException()); + if (rh.compareAndSet(r, nr)) { + // success + sendResponseIfRequired(subscription, nr); + break; + } else { + // we'll retry + } + } + return subscription; + } + + private static void sendResponseIfRequired(BooleanSubscription subscription, CollapsedRequestObservableFunction.ResponseHolder r) { + if (!subscription.isUnsubscribed()) { + Observer o = r.getObserver(); + if (o == null || (r.getException() == null && r.getResponse() == null)) { + // not ready to send + return; + } + + if (r.getException() != null) { + o.onError(r.getException()); + } else { + o.onNext(r.getResponse()); + o.onCompleted(); + } + } + } + + /** + * Used for atomic compound updates. + */ + private static class ResponseHolder { + private final T response; + private final Exception e; + private final Observer observer; + + public ResponseHolder() { + this(null, null, null); + } + + public ResponseHolder(Observer observer, T response, Exception e) { + this.observer = observer; + this.response = response; + this.e = e; + } + + public Observer getObserver() { + return observer; + } + + public T getResponse() { + return response; + } + + public Exception getException() { + return e; + } + + } + + public static class UnitTest { + + @Test + public void testSetResponseSuccess() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + // fetch value + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetException() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setException(new RuntimeException("anException")); + + // fetch value + try { + v.get(); + fail("expected exception"); + } catch (ExecutionException e) { + assertEquals("anException", e.getCause().getMessage()); + } + } + + @Test + public void testSetExceptionAfterResponse() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + try { + cr.setException(new RuntimeException("anException")); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetResponseAfterException() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setException(new RuntimeException("anException")); + + try { + cr.setResponse("theResponse"); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + try { + v.get(); + fail("expected exception"); + } catch (ExecutionException e) { + assertEquals("anException", e.getCause().getMessage()); + } + } + + @Test + public void testSetResponseDuplicate() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + try { + cr.setResponse("theResponse2"); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetResponseAfterUnsubscribe() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future f = o.toBlockingObservable().toFuture(); + + // cancel/unsubscribe + f.cancel(true); + + try { + cr.setResponse("theResponse"); + } catch (IllegalStateException e) { + fail("this should have done nothing as it was unsubscribed already"); + } + + // if you fetch after canceling it should be null + assertEquals(null, f.get()); + } + + @Test + public void testSetExceptionAfterUnsubscribe() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future f = o.toBlockingObservable().toFuture(); + + // cancel/unsubscribe + f.cancel(true); + + try { + cr.setException(new RuntimeException("anException")); + } catch (IllegalStateException e) { + fail("this should have done nothing as it was unsubscribed already"); + } + + // if you fetch after canceling it should be null + assertEquals(null, f.get()); + } + + @Test + public void testUnsubscribeAfterSetResponse() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + // unsubscribe after the value is sent + v.cancel(true); + + // still get value as it was set before canceling + assertEquals("theResponse", v.get()); + } + + } + +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapserTimer.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapserTimer.java new file mode 100644 index 000000000..a252f1fbc --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapserTimer.java @@ -0,0 +1,13 @@ +package com.netflix.hystrix.collapser; + +import java.lang.ref.Reference; + +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +/** + * Timer used for trigger batch execution. + */ +public interface CollapserTimer { + + public Reference addListener(TimerListener collapseTask); +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/HystrixCollapserBridge.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/HystrixCollapserBridge.java new file mode 100644 index 000000000..98c2914a4 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/HystrixCollapserBridge.java @@ -0,0 +1,27 @@ +package com.netflix.hystrix.collapser; + +import java.util.Collection; + +import rx.Observable; + +import com.netflix.hystrix.HystrixCollapser.CollapsedRequest; +import com.netflix.hystrix.HystrixCollapserKey; + +/** + * Bridge between HystrixCollapser and RequestCollapser to expose 'protected' and 'private' functionality across packages. + * + * @param + * @param + * @param + */ +public interface HystrixCollapserBridge { + + public Collection>> shardRequests(Collection> requests); + + public Observable createObservableCommand(Collection> requests); + + public void mapResponseToRequests(BatchReturnType batchResponse, Collection> requests); + + public HystrixCollapserKey getCollapserKey(); + +} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/README.txt b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/README.txt new file mode 100644 index 000000000..77d2c0acb --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/README.txt @@ -0,0 +1,3 @@ +This package is not part of the public API and can change at any time. Do not rely upon any classes in this package. + +The public API is com.netflix.hystrix.HystrixCollapser \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RealCollapserTimer.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RealCollapserTimer.java new file mode 100644 index 000000000..9188964d0 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RealCollapserTimer.java @@ -0,0 +1,20 @@ +package com.netflix.hystrix.collapser; + +import java.lang.ref.Reference; + +import com.netflix.hystrix.util.HystrixTimer; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +/** + * Actual CollapserTimer implementation for triggering batch execution that uses HystrixTimer. + */ +public class RealCollapserTimer implements CollapserTimer { + /* single global timer that all collapsers will schedule their tasks on */ + private final static HystrixTimer timer = HystrixTimer.getInstance(); + + @Override + public Reference addListener(TimerListener collapseTask) { + return timer.addTimerListener(collapseTask); + } + +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java new file mode 100644 index 000000000..12a6fe988 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java @@ -0,0 +1,238 @@ +package com.netflix.hystrix.collapser; + +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.Observer; + +import com.netflix.hystrix.HystrixCollapser; +import com.netflix.hystrix.HystrixCollapser.CollapsedRequest; +import com.netflix.hystrix.HystrixCollapserProperties; + +/** + * A batch of requests collapsed together by a RequestCollapser instance. When full or time has expired it will execute and stop accepting further submissions. + * + * @param + * @param + * @param + */ +public class RequestBatch { + + private static final Logger logger = LoggerFactory.getLogger(HystrixCollapser.class); + + private final HystrixCollapserBridge commandCollapser; + final ConcurrentLinkedQueue> requests = new ConcurrentLinkedQueue>(); + // use AtomicInteger to count so we can use ConcurrentLinkedQueue instead of LinkedBlockingQueue + private final AtomicInteger count = new AtomicInteger(0); + private final int maxBatchSize; + private final AtomicBoolean batchStarted = new AtomicBoolean(); + + private ReentrantReadWriteLock batchLock = new ReentrantReadWriteLock(); + + public RequestBatch(HystrixCollapserProperties properties, HystrixCollapserBridge commandCollapser, int maxBatchSize) { + this.commandCollapser = commandCollapser; + this.maxBatchSize = maxBatchSize; + } + + /** + * @return Observable if offer accepted, null if batch is full, already started or completed + */ + public Observable offer(RequestArgumentType arg) { + /* short-cut - if the batch is started we reject the offer */ + if (batchStarted.get()) { + return null; + } + + /* + * The 'read' just means non-exclusive even though we are writing. + */ + if (batchLock.readLock().tryLock()) { + try { + /* double-check now that we have the lock - if the batch is started we reject the offer */ + if (batchStarted.get()) { + return null; + } + + int s = count.incrementAndGet(); + if (s > maxBatchSize) { + return null; + } else { + CollapsedRequestObservableFunction f = new CollapsedRequestObservableFunction(arg); + requests.add(f); + return Observable.create(f); + } + } finally { + batchLock.readLock().unlock(); + } + } else { + return null; + } + } + + /** + * Collapsed requests are triggered for batch execution and the array of arguments is passed in. + *

+ * IMPORTANT IMPLEMENTATION DETAILS => The expected contract (responsibilities) of this method implementation is: + *

+ *

    + *
  • Do NOT block => Do the work on a separate worker thread. Do not perform inline otherwise it will block other requests.
    • + *
  • Set ALL CollapsedRequest response values => Set the response values T on each CollapsedRequest, even if the response is NULL otherwise the user thread waiting on the response will + * think a response was never received and will either block indefinitely or will timeout while waiting.
    • + *
+ * + * @param args + */ + public void executeBatchIfNotAlreadyStarted() { + /* + * - check that we only execute once since there's multiple paths to do so (timer, waiting thread or max batch size hit) + * - close the gate so 'offer' can no longer be invoked and we turn those threads away so they create a new batch + */ + if (batchStarted.compareAndSet(false, true)) { + /* wait for 'offer' threads to finish before executing the batch so 'requests' is complete */ + batchLock.writeLock().lock(); + try { + // shard batches + Collection>> shards = commandCollapser.shardRequests(requests); + // for each shard execute its requests + for (final Collection> shardRequests : shards) { + try { + // create a new command to handle this batch of requests + Observable o = commandCollapser.createObservableCommand(shardRequests); + o.subscribe(new RequestBatch.BatchRequestObserver(commandCollapser, shardRequests)); + } catch (Exception e) { + logger.error("Exception while creating and queueing command with batch.", e); + // if a failure occurs we want to pass that exception to all of the Futures that we've returned + for (CollapsedRequest request : shardRequests) { + try { + request.setException(e); + } catch (IllegalStateException e2) { + logger.debug("Failed trying to setException on CollapsedRequest", e2); + } + } + } + } + + } catch (Exception e) { + logger.error("Exception while sharding requests.", e); + // same error handling as we do around the shards, but this is a wider net in case the shardRequest method fails + for (CollapsedRequest request : requests) { + try { + request.setException(e); + } catch (IllegalStateException e2) { + logger.debug("Failed trying to setException on CollapsedRequest", e2); + } + } + } finally { + batchLock.writeLock().unlock(); + } + } + } + + public void shutdown() { + // take the 'batchStarted' state so offers and execution will not be triggered elsewhere + if (batchStarted.compareAndSet(false, true)) { + // get the write lock so offers are synced with this (we don't really need to unlock as this is a one-shot deal to shutdown) + batchLock.writeLock().lock(); + try { + // if we win the 'start' and once we have the lock we can now shut it down otherwise another thread will finish executing this batch + if (requests.size() > 0) { + logger.warn("Requests still exist in queue but will not be executed due to RequestCollapser shutdown: " + requests.size(), new IllegalStateException()); + /* + * In the event that there is a concurrency bug or thread scheduling prevents the timer from ticking we need to handle this so the Future.get() calls do not block. + * + * I haven't been able to reproduce this use case on-demand but when stressing a machine saw this occur briefly right after the JVM paused (logs stopped scrolling). + * + * This safety-net just prevents the CollapsedRequestFutureImpl.get() from waiting on the CountDownLatch until its max timeout. + */ + for (CollapsedRequest request : requests) { + try { + ((CollapsedRequestObservableFunction) request).setExceptionIfResponseNotReceived(new IllegalStateException("Requests not executed before shutdown.")); + } catch (Exception e) { + logger.debug("Failed to setException on CollapsedRequestFutureImpl instances.", e); + } + /** + * https://github.com/Netflix/Hystrix/issues/78 Include more info when collapsed requests remain in queue + */ + logger.warn("Request still in queue but not be executed due to RequestCollapser shutdown. Argument => " + request.getArgument() + " Request Object => " + request, new IllegalStateException()); + } + + } + } finally { + batchLock.writeLock().unlock(); + } + } + } + + private static final class BatchRequestObserver implements Observer { + private final Collection> requests; + private final HystrixCollapserBridge commandCollapser; + + private BatchRequestObserver(HystrixCollapserBridge commandCollapser, Collection> requests) { + this.commandCollapser = commandCollapser; + this.requests = requests; + } + + @Override + public void onCompleted() { + // do nothing as we always expect onNext or onError to be called + } + + @Override + public void onError(Exception e) { + logger.error("Exception while executing command with batch.", e); + // if a failure occurs we want to pass that exception to all of the Futures that we've returned + for (CollapsedRequest request : requests) { + try { + request.setException(e); + } catch (IllegalStateException e2) { + logger.debug("Failed trying to setException on CollapsedRequest", e2); + } + } + } + + @Override + public void onNext(BatchReturnType response) { + try { + commandCollapser.mapResponseToRequests(response, requests); + } catch (Throwable e) { + // handle Throwable in case anything is thrown so we don't block Observers waiting for onError/onCompleted + Exception ee = null; + if (e instanceof Exception) { + ee = (Exception) e; + } else { + ee = new RuntimeException("Throwable caught while invoking 'mapResponseToRequests'", e); + } + logger.error("Exception mapping responses to requests.", e); + // if a failure occurs we want to pass that exception to all of the Futures that we've returned + for (CollapsedRequest request : requests) { + try { + ((CollapsedRequestObservableFunction) request).setExceptionIfResponseNotReceived(ee); + } catch (IllegalStateException e2) { + // if we have partial responses set in mapResponseToRequests + // then we may get IllegalStateException as we loop over them + // so we'll log but continue to the rest + logger.error("Partial success of 'mapResponseToRequests' resulted in IllegalStateException while setting Exception. Continuing ... ", e2); + } + } + } + + // check that all requests had setResponse or setException invoked in case 'mapResponseToRequests' was implemented poorly + IllegalStateException ie = new IllegalStateException("No response set by " + commandCollapser.getCollapserKey().name() + " 'mapResponseToRequests' implementation."); + for (CollapsedRequest request : requests) { + try { + ((CollapsedRequestObservableFunction) request).setExceptionIfResponseNotReceived(ie); + } catch (IllegalStateException e2) { + logger.debug("Partial success of 'mapResponseToRequests' resulted in IllegalStateException while setting 'No response set' Exception. Continuing ... ", e2); + } + } + } + } + +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapser.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapser.java new file mode 100644 index 000000000..f5ecf70ed --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapser.java @@ -0,0 +1,167 @@ +package com.netflix.hystrix.collapser; + +import java.lang.ref.Reference; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.concurrent.ThreadSafe; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; + +import com.netflix.hystrix.HystrixCollapser; +import com.netflix.hystrix.HystrixCollapserProperties; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixContextCallable; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +/** + * Requests are submitted to this and batches executed based on size or time. Scoped to either a request or the global application. + *

+ * Instances of this are retrieved from the RequestCollapserFactory. + * + * Must be thread-safe since it exists within a RequestVariable which is request-scoped and can be accessed from multiple threads. + */ +@ThreadSafe +public class RequestCollapser { + static final Logger logger = LoggerFactory.getLogger(HystrixCollapser.class); + + private final HystrixCollapserBridge commandCollapser; + // batch can be null once shutdown + private final AtomicReference> batch = new AtomicReference>(); + private final AtomicReference> timerListenerReference = new AtomicReference>(); + private final AtomicBoolean timerListenerRegistered = new AtomicBoolean(); + private final CollapserTimer timer; + private final HystrixCollapserProperties properties; + private final HystrixConcurrencyStrategy concurrencyStrategy; + + /** + * @param maxRequestsInBatch + * Maximum number of requests to include in a batch. If request count hits this threshold it will result in batch executions earlier than the scheduled delay interval. + * @param timerDelayInMilliseconds + * Interval between batch executions. + * @param commandCollapser + */ + RequestCollapser(HystrixCollapserBridge commandCollapser, HystrixCollapserProperties properties, CollapserTimer timer, HystrixConcurrencyStrategy concurrencyStrategy) { + this.commandCollapser = commandCollapser; // the command with implementation of abstract methods we need + this.concurrencyStrategy = concurrencyStrategy; + this.properties = properties; + this.timer = timer; + batch.set(new RequestBatch(properties, commandCollapser, properties.maxRequestsInBatch().get())); + } + + /** + * Submit a request to a batch. If the batch maxSize is hit trigger the batch immediately. + * + * @param arg + * @return Observable + * @throws IllegalStateException + * if submitting after shutdown + */ + public Observable submitRequest(RequestArgumentType arg) { + /* + * We only want the timer ticking if there are actually things to do so we register it the first time something is added. + */ + if (!timerListenerRegistered.get() && timerListenerRegistered.compareAndSet(false, true)) { + /* schedule the collapsing task to be executed every x milliseconds (x defined inside CollapsedTask) */ + timerListenerReference.set(timer.addListener(new CollapsedTask())); + } + + // loop until succeed (compare-and-set spin-loop) + while (true) { + RequestBatch b = batch.get(); + if (b == null) { + throw new IllegalStateException("Submitting requests after collapser is shutdown"); + } + Observable f = b.offer(arg); + // it will always get an Observable unless we hit the max batch size + if (f != null) { + return f; + } else { + // this batch can't accept requests so create a new one and set it if another thread doesn't beat us + createNewBatchAndExecutePreviousIfNeeded(b); + } + } + } + + private void createNewBatchAndExecutePreviousIfNeeded(RequestBatch previousBatch) { + if (previousBatch == null) { + throw new IllegalStateException("Trying to start null batch which means it was shutdown already."); + } + if (batch.compareAndSet(previousBatch, new RequestBatch(properties, commandCollapser, properties.maxRequestsInBatch().get()))) { + // this thread won so trigger the previous batch + previousBatch.executeBatchIfNotAlreadyStarted(); + } + } + + /** + * Called from RequestVariable.shutdown() to unschedule the task. + */ + public void shutdown() { + RequestBatch currentBatch = batch.getAndSet(null); + if (currentBatch != null) { + currentBatch.shutdown(); + } + + if (timerListenerReference.get() != null) { + // if the timer was started we'll clear it so it stops ticking + timerListenerReference.get().clear(); + } + } + + /** + * Executed on each Timer interval execute the current batch if it has requests in it. + */ + private class CollapsedTask implements TimerListener { + final Callable callableWithContextOfParent; + + CollapsedTask() { + // this gets executed from the context of a HystrixCommand parent thread (such as a Tomcat thread) + // so we create the callable now where we can capture the thread context + callableWithContextOfParent = concurrencyStrategy.wrapCallable(new HystrixContextCallable(new Callable() { + // the wrapCallable call allows a strategy to capture thread-context if desired + + @Override + public Void call() throws Exception { + try { + // we fetch current so that when multiple threads race + // we can do compareAndSet with the expected/new to ensure only one happens + RequestBatch currentBatch = batch.get(); + // 1) it can be null if it got shutdown + // 2) we don't execute this batch if it has no requests and let it wait until next tick to be executed + if (currentBatch != null && currentBatch.requests.size() > 0) { + // do execution within context of wrapped Callable + createNewBatchAndExecutePreviousIfNeeded(currentBatch); + } + } catch (Throwable t) { + logger.error("Error occurred trying to executeRequestsFromQueue.", t); + // ignore error so we don't kill the Timer mainLoop and prevent further items from being scheduled + // http://jira.netflix.com/browse/API-5042 HystrixCommand: Collapser TimerThread Vulnerable to Shutdown + } + return null; + } + + })); + } + + @Override + public void tick() { + try { + callableWithContextOfParent.call(); + } catch (Exception e) { + logger.error("Error occurred trying to execute callable inside CollapsedTask from Timer.", e); + e.printStackTrace(); + } + } + + @Override + public int getIntervalTimeInMilliseconds() { + return properties.timerDelayInMilliseconds().get(); + } + + } + +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapserFactory.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapserFactory.java new file mode 100644 index 000000000..0129fc780 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestCollapserFactory.java @@ -0,0 +1,266 @@ +package com.netflix.hystrix.collapser; + +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.hystrix.HystrixCollapser; +import com.netflix.hystrix.HystrixCollapser.Scope; +import com.netflix.hystrix.HystrixCollapserKey; +import com.netflix.hystrix.HystrixCollapserProperties; +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableHolder; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesFactory; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.util.HystrixTimer; + +/** + * Factory for retrieving the correct instance of a RequestCollapser. + * + * @param + * @param + * @param + */ +public class RequestCollapserFactory { + + private static final Logger logger = LoggerFactory.getLogger(RequestCollapserFactory.class); + + private final CollapserTimer timer; + private final HystrixCollapserKey collapserKey; + private final HystrixCollapserProperties properties; + private final HystrixConcurrencyStrategy concurrencyStrategy; + private final Scope scope; + + public RequestCollapserFactory(HystrixCollapserKey collapserKey, Scope scope, CollapserTimer timer, HystrixCollapserProperties.Setter propertiesBuilder) { + /* strategy: ConcurrencyStrategy */ + this.concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); + + this.timer = timer; + this.scope = scope; + this.collapserKey = collapserKey; + this.properties = HystrixPropertiesFactory.getCollapserProperties(this.collapserKey, propertiesBuilder); + } + + public HystrixCollapserKey getCollapserKey() { + return collapserKey; + } + + public Scope getScope() { + return scope; + } + + public HystrixCollapserProperties getProperties() { + return properties; + } + + public RequestCollapser getRequestCollapser(HystrixCollapserBridge commandCollapser) { + if (Scope.REQUEST == getScope()) { + return getCollapserForUserRequest(commandCollapser); + } else if (Scope.GLOBAL == getScope()) { + return getCollapserForGlobalScope(commandCollapser); + } else { + logger.warn("Invalid Scope: " + getScope() + " Defaulting to REQUEST scope."); + return getCollapserForUserRequest(commandCollapser); + } + } + + /** + * Static global cache of RequestCollapsers for Scope.GLOBAL + */ + // String is CollapserKey.name() (we can't use CollapserKey directly as we can't guarantee it implements hashcode/equals correctly) + private static ConcurrentHashMap> globalScopedCollapsers = new ConcurrentHashMap>(); + + @SuppressWarnings("unchecked") + private RequestCollapser getCollapserForGlobalScope(HystrixCollapserBridge commandCollapser) { + RequestCollapser collapser = globalScopedCollapsers.get(collapserKey.name()); + if (collapser != null) { + return (RequestCollapser) collapser; + } + // create new collapser using 'this' first instance as the one that will get cached for future executions ('this' is stateless so we can do that) + RequestCollapser newCollapser = new RequestCollapser(commandCollapser, properties, timer, concurrencyStrategy); + RequestCollapser existing = globalScopedCollapsers.putIfAbsent(collapserKey.name(), newCollapser); + if (existing == null) { + // we won + return newCollapser; + } else { + // we lost ... another thread beat us + // shutdown the one we created but didn't get stored + newCollapser.shutdown(); + // return the existing one + return (RequestCollapser) existing; + } + } + + /** + * Static global cache of RequestVariables with RequestCollapsers for Scope.REQUEST + */ + // String is HystrixCollapserKey.name() (we can't use HystrixCollapserKey directly as we can't guarantee it implements hashcode/equals correctly) + private static ConcurrentHashMap>> requestScopedCollapsers = new ConcurrentHashMap>>(); + + /* we are casting because the Map needs to be but we know it is for this thread */ + @SuppressWarnings("unchecked") + private RequestCollapser getCollapserForUserRequest(HystrixCollapserBridge commandCollapser) { + return (RequestCollapser) getRequestVariableForCommand(commandCollapser).get(concurrencyStrategy); + } + + /** + * Lookup (or create and store) the RequestVariable for a given HystrixCollapserKey. + * + * @param key + * @return HystrixRequestVariableHolder + */ + @SuppressWarnings("unchecked") + private HystrixRequestVariableHolder> getRequestVariableForCommand(final HystrixCollapserBridge commandCollapser) { + HystrixRequestVariableHolder> requestVariable = requestScopedCollapsers.get(commandCollapser.getCollapserKey().name()); + if (requestVariable == null) { + // create new collapser using 'this' first instance as the one that will get cached for future executions ('this' is stateless so we can do that) + @SuppressWarnings({ "rawtypes" }) + HystrixRequestVariableHolder newCollapser = new RequestCollapserRequestVariable(commandCollapser, properties, timer, concurrencyStrategy); + HystrixRequestVariableHolder> existing = requestScopedCollapsers.putIfAbsent(commandCollapser.getCollapserKey().name(), newCollapser); + if (existing == null) { + // this thread won, so return the one we just created + requestVariable = newCollapser; + } else { + // another thread beat us (this should only happen when we have concurrency on the FIRST request for the life of the app for this HystrixCollapser class) + requestVariable = existing; + /* + * This *should* be okay to discard the created object without cleanup as the RequestVariable implementation + * should properly do lazy-initialization and only call initialValue() the first time get() is called. + * + * If it does not correctly follow this contract then there is a chance of a memory leak here. + */ + } + } + return requestVariable; + } + + /** + * Clears all state. If new requests come in instances will be recreated and metrics started from scratch. + */ + public static void reset() { + defaultNameCache.clear(); + globalScopedCollapsers.clear(); + requestScopedCollapsers.clear(); + HystrixTimer.reset(); + } + + /** + * Used for testing + */ + public static void resetRequest() { + requestScopedCollapsers.clear(); + } + + /** + * Used for testing + */ + public static HystrixRequestVariableHolder> getRequestVariable(String key) { + return requestScopedCollapsers.get(key); + } + + /** + * Request scoped RequestCollapser that lives inside a RequestVariable. + *

+ * This depends on the RequestVariable getting reset before each user request in NFFilter to ensure the RequestCollapser is new for each user request. + */ + private final class RequestCollapserRequestVariable extends HystrixRequestVariableHolder> { + + /** + * NOTE: There is only 1 instance of this for the life of the app per HystrixCollapser instance. The state changes on each request via the initialValue()/get() methods. + *

+ * Thus, do NOT put any instance variables in this class that are not static for all threads. + */ + + private RequestCollapserRequestVariable(final HystrixCollapserBridge commandCollapser, final HystrixCollapserProperties properties, final CollapserTimer timer, final HystrixConcurrencyStrategy concurrencyStrategy) { + super(new HystrixRequestVariableLifecycle>() { + @Override + public RequestCollapser initialValue() { + // this gets calls once per request per HystrixCollapser instance + return new RequestCollapser(commandCollapser, properties, timer, concurrencyStrategy); + } + + @Override + public void shutdown(RequestCollapser currentCollapser) { + // shut down the RequestCollapser (the internal timer tasks) + if (currentCollapser != null) { + currentCollapser.shutdown(); + } + } + }); + } + + } + + /** + * Fluent interface for arguments to the {@link HystrixCollapser} constructor. + *

+ * The required arguments are set via the 'with' factory method and optional arguments via the 'and' chained methods. + *

+ * Example: + * 

 {@code
+     *  Setter.withCollapserKey(HystrixCollapserKey.Factory.asKey("CollapserName"))
+                .andScope(Scope.REQUEST);
+     * }
+ */ + @NotThreadSafe + public static class Setter { + private final HystrixCollapserKey collapserKey; + private Scope scope = Scope.REQUEST; // default if nothing is set + private HystrixCollapserProperties.Setter propertiesSetter; + + private Setter(HystrixCollapserKey collapserKey) { + this.collapserKey = collapserKey; + } + + /** + * Setter factory method containing required values. + *

+ * All optional arguments can be set via the chained methods. + * + * @param collapserKey + * {@link HystrixCollapserKey} that identifies this collapser and provides the key used for retrieving properties, request caches, publishing metrics etc. + * @return Setter for fluent interface via method chaining + */ + public static Setter withCollapserKey(HystrixCollapserKey collapserKey) { + return new Setter(collapserKey); + } + + /** + * {@link Scope} defining what scope the collapsing should occur within + * + * @param scope + * + * @return Setter for fluent interface via method chaining + */ + public Setter andScope(Scope scope) { + this.scope = scope; + return this; + } + + /** + * @param propertiesSetter + * {@link HystrixCollapserProperties.Setter} that allows instance specific property overrides (which can then be overridden by dynamic properties, see + * {@link HystrixPropertiesStrategy} for + * information on order of precedence). + *

+ * Will use defaults if left NULL. + * @return Setter for fluent interface via method chaining + */ + public Setter andCollapserPropertiesDefaults(HystrixCollapserProperties.Setter propertiesSetter) { + this.propertiesSetter = propertiesSetter; + return this; + } + + } + + // this is a micro-optimization but saves about 1-2microseconds (on 2011 MacBook Pro) + // on the repetitive string processing that will occur on the same classes over and over again + @SuppressWarnings("rawtypes") + private static ConcurrentHashMap, String> defaultNameCache = new ConcurrentHashMap, String>(); + +} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java index eba34c8f5..0c14b6e57 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java @@ -31,9 +31,10 @@ import org.slf4j.LoggerFactory; import com.netflix.hystrix.HystrixCollapser; +import com.netflix.hystrix.HystrixCommand; /** - * Timer used by the {@link HystrixCollapser} to trigger batch executions. + * Timer used by {@link HystrixCommand} to timeout async executions and {@link HystrixCollapser} to trigger batch executions. */ public class HystrixTimer { @@ -46,7 +47,7 @@ private HystrixTimer() { } /** - * Retrieve the global instance with a single backing thread. + * Retrieve the global instance. */ public static HystrixTimer getInstance() { return INSTANCE; diff --git a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java index 175031d41..85f36573e 100644 --- a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java +++ b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java @@ -98,15 +98,18 @@ public void testCollapser() throws Exception { assertEquals("ValueForKey: 3", f3.get()); assertEquals("ValueForKey: 4", f4.get()); - // assert that the batch command 'GetValueForKey' was in fact - // executed and that it executed only once - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; - // assert the command is the one we're expecting - assertEquals("GetValueForKey", command.getCommandKey().name()); - // confirm that it was a COLLAPSED command execution - assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + // assert that the batch command 'GetValueForKey' was in fact executed and that it executed only + // once or twice (due to non-determinism of scheduler since this example uses the real timer) + if (HystrixRequestLog.getCurrentRequest().getExecutedCommands().size() > 2) { + fail("some of the commands should have been collapsed"); + } + for (HystrixCommand command : HystrixRequestLog.getCurrentRequest().getExecutedCommands()) { + // assert the command is the one we're expecting + assertEquals("GetValueForKey", command.getCommandKey().name()); + // confirm that it was a COLLAPSED command execution + assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + } } finally { context.shutdown(); } diff --git a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandWithFallbackViaNetwork.java b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandWithFallbackViaNetwork.java index 20b35f6b6..394df173d 100644 --- a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandWithFallbackViaNetwork.java +++ b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandWithFallbackViaNetwork.java @@ -92,11 +92,12 @@ public void test() { HystrixRequestContext context = HystrixRequestContext.initializeContext(); try { assertEquals(null, new CommandWithFallbackViaNetwork(1).execute()); - HystrixCommand command1 = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[1]; + + HystrixCommand command1 = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[0]; assertEquals("GetValueCommand", command1.getCommandKey().name()); assertTrue(command1.getExecutionEvents().contains(HystrixEventType.FAILURE)); - HystrixCommand command2 = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[0]; + HystrixCommand command2 = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[1]; assertEquals("GetValueFallbackCommand", command2.getCommandKey().name()); assertTrue(command2.getExecutionEvents().contains(HystrixEventType.FAILURE)); } finally {