Improved reporting and error handling

docs/src/jekyll/_config.yml

@@ -6,6 +6,8 @@ style: fetch
 highlight_theme: tomorrow
 docs: true
 markdown: redcarpet
+cdn:
+  url: https://rawgit.com/47deg/microsites/cdn/
 collections:
   tut:
     output: true

docs/src/tut/docs.md

@@ -126,6 +126,8 @@ val userDatabase: Map[UserId, User] = Map(
 )
 
 implicit object UserSource extends DataSource[UserId, User]{
+  override def name = "User data source"
+
   override def fetchOne(id: UserId): Query[Option[User]] = {
     Query.sync({
 	  latency(userDatabase.get(id), s"One User $id")
@@ -336,6 +338,8 @@ val postDatabase: Map[PostId, Post] = Map(
 )
 
 implicit object PostSource extends DataSource[PostId, Post]{
+  override def name = "Post data source"
+
   override def fetchOne(id: PostId): Query[Option[Post]] = {
     Query.sync({
 	  latency(postDatabase.get(id), s"One Post $id")
@@ -367,6 +371,8 @@ We'll implement a data source for retrieving a post topic given a post id.
 
 ```tut:silent
 implicit object PostTopicSource extends DataSource[Post, PostTopic]{
+  override def name = "Post topic data source"
+
   override def fetchOne(id: Post): Query[Option[PostTopic]] = {
     Query.sync({
       val topic = if (id.id % 2 == 0) "monad" else "applicative"
@@ -575,13 +581,25 @@ fetchSameTwice.runA[Id](ForgetfulCache())
 
 # Error handling
 
-Fetch is used for reading data from remote sources and the queries we perform can and will fail at some point. What happens if we run a fetch and fails? We'll create a fetch that always fails to learn about it.
+Fetch is used for reading data from remote sources and the queries we perform can and will fail at some point. There are many things that can
+go wrong:
+ - an exception can be thrown by client code of certain data sources
+ - an identity may be missing
+ - the data source may be temporarily available
+
+Since the error cases are plenty and can't be anticipated Fetch errors are represented by the `FetchError` trait, which extends `Throwable`.
+Currently fetch defines `FetchError` cases for missing identities and arbitrary exceptions but you can extend `FetchError` with any error
+you want.
+
+## Exceptions
+
+What happens if we run a fetch and fails with an exception? We'll create a fetch that always fails to learn about it.
 
 ```tut:silent
 val fetchError: Fetch[User] = (new Exception("Oh noes")).fetch
 ```
 
-If we try to execute to `Id` the exception will be thrown.
+If we try to execute to `Id` the exception will be thrown wrapped in a `FetchException`.
 
 ```tut:fail
 fetchError.runA[Id]
@@ -596,13 +614,13 @@ We can use the `FetchMonadError[Eval]#attempt` to convert a fetch result into a
 import fetch.unsafe.implicits._
 ```
 
-Now we can convert `Eval[User]` into `Eval[Throwable Xor User]` and capture exceptions as values in the left of the disjunction.
+Now we can convert `Eval[User]` into `Eval[FetchError Xor User]` and capture exceptions as values in the left of the disjunction.
 
 ```tut:book
 import cats.Eval
 import cats.data.Xor
 
-val safeResult: Eval[Throwable Xor User] = FetchMonadError[Eval].attempt(fetchError.runA[Eval])
+val safeResult: Eval[FetchError Xor User] = FetchMonadError[Eval].attempt(fetchError.runA[Eval])
 
 safeResult.value
 ```
@@ -617,12 +635,61 @@ fetchError.runA[Eval].attempt.value
 
 ## Missing identities
 
-You've probably noticed that `DataSource.fetch` takes a list of identities and returns a map of identities to their result, taking
-into account the possibility of some identities not being found. Whenever an identity cannot be found, the fetch execution will
-fail.
+You've probably noticed that `DataSource.fetchOne` and `DataSource.fetchMany` return types help Fetch know if any requested
+identity was not found. Whenever an identity cannot be found, the fetch execution will fail with an instance of `FetchError`.
+
+The requests can be of different types, each of which is described below.
+
+### One request
+
+When a single identity is being fetched the request will be a `FetchOne`; it contains the data source and the identity to fetch so you
+should be able to easily diagnose the failure. For ilustrating this scenario we'll ask for users that are not in the database.
+
+```tut:silent
+val missingUser = getUser(5)
+val eval: Eval[FetchError Xor User] = missingUser.runA[Eval].attempt
+val result: FetchError Xor User = eval.value
+val err: FetchError = result.swap.toOption.get // don't do this at home, folks
+```
+
+`NotFound` allows you to access the fetch request that was in progress when the error happened and the environment of the fetch.
+
+```tut:book
+err match {
+  case nf: NotFound => {
+    println("Request " + nf.request)
+    println("Environment " + nf.env)
+  }
+}
+```
+
+As you can see in the output, the error was actually a `NotFound`. We can access the request with `.request`, which lets us
+know that the failed request was for the identity `5` of the user data source. We can also see that the environment has an empty
+cache and no rounds of execution happened yet.
+
+### Multiple requests
+
+When multiple requests to the same data source are batched and/or multiple requests are performed at the same time, is possible that more than one identity was missing. There is another error case for such situations: `MissingIdentities`, which contains a mapping from data source names to the list of missing identities.
+
+```tut:silent
+val missingUsers = List(3, 4, 5, 6).traverse(getUser)
+val eval: Eval[FetchError Xor List[User]] = missingUsers.runA[Eval].attempt
+val result: FetchError Xor List[User] = eval.value
+val err: FetchError = result.swap.toOption.get // don't do this at home, folks
+```
+
+The `.missing` attribute will give us the mapping from data source name to missing identities, and `.env` will give us the environment so we can track the execution of the fetch.
+
+```tut:book
+err match {
+  case mi: MissingIdentities => {
+    println("Missing identities " + mi.missing)
+    println("Environment " + mi.env)
+  }
+}
+```
 
-Whenever a fetch fails, a `FetchFailure` exception is thrown. The `FetchFailure` will have the environment, which gives you information
-about the execution of the fetch.
+## Your own errors
 
 # Syntax
 
@@ -907,7 +974,7 @@ Await.result(task.runAsync(ioSched), Duration.Inf)
 
 ## Custom types
 
-If you want to run a fetch to a custom type `M[_]`, you need to implement the `FetchMonadError[M]` typeclass. `FetchMonadError[M]` is simply a `MonadError[M, Throwable]` from cats augmented
+If you want to run a fetch to a custom type `M[_]`, you need to implement the `FetchMonadError[M]` typeclass. `FetchMonadError[M]` is simply a `MonadError[M, FetchError]` from cats augmented
 with a method for running a `Query[A]` in the context of the monad `M[A]`.
 
 For ilustrating integration with an asynchronous concurrency monad we'll use the implementation of Monix Task.
@@ -983,10 +1050,10 @@ implicit val taskFetchMonadError: FetchMonadError[Task] = new FetchMonadError[Ta
   def pure[A](x: A): Task[A] =
     Task.now(x)
 
-  def handleErrorWith[A](fa: Task[A])(f: Throwable => Task[A]): Task[A] =
-    fa.onErrorHandleWith(f)
+  def handleErrorWith[A](fa: Task[A])(f: FetchError => Task[A]): Task[A] =
+    fa.onErrorHandleWith({ case e: FetchError => f(e) })
 
-  def raiseError[A](e: Throwable): Task[A] =
+  def raiseError[A](e: FetchError): Task[A] =
     Task.raiseError(e)
 
   def flatMap[A, B](fa: Task[A])(f: A => Task[B]): Task[B] =
@@ -1016,7 +1083,7 @@ Await.result(task.runAsync(scheduler), Duration.Inf)
 Fetch stands on the shoulders of giants:
 
 - [Haxl](https://github.com/facebook/haxl) is Facebook's implementation (Haskell) of the [original paper Fetch is based on](http://community.haskell.org/~simonmar/papers/haxl-icfp14.pdf).
-- [Clump](http://getclump.io) has inspired the signature of the `DataSource#fetch` method.
+- [Clump](http://getclump.io) has inspired the signature of the `DataSource#fetch*` methods.
 - [Stitch](https://engineering.twitter.com/university/videos/introducing-stitch) is an in-house Twitter library that is not open source but has inspired Fetch's high-level API.
 - [Cats](http://typelevel.org/cats/), a library for functional programming in Scala.
 - [Monix](https://monix.io) high-performance and multiplatform (Scala / Scala.js) asynchronous programming library.

jvm/src/main/scala/unsafeImplicits.scala

@@ -53,15 +53,22 @@ object implicits {
         }
     }
     def pure[A](x: A): Eval[A] = Eval.now(x)
-    def handleErrorWith[A](fa: Eval[A])(f: Throwable => Eval[A]): Eval[A] =
+
+    def handleErrorWith[A](fa: Eval[A])(f: FetchError => Eval[A]): Eval[A] =
       Eval.later({
         try {
           fa.value
         } catch {
-          case ex: Throwable => f(ex).value
+          case ex: FetchError => f(ex).value
+          case th: Throwable  => f(FetchException(th)).value
         }
       })
-    def raiseError[A](e: Throwable): Eval[A] = Eval.later({ throw e })
+
+    def raiseError[A](e: FetchError): Eval[A] =
+      Eval.later({
+        throw e
+      })
+
     def flatMap[A, B](fa: Eval[A])(f: A => Eval[B]): Eval[B] =
       fa.flatMap(f)
   }
@@ -93,13 +100,20 @@ object implicits {
         }
     }
     def pure[A](x: A): Id[A] = x
-    def handleErrorWith[A](fa: Id[A])(f: Throwable => Id[A]): Id[A] =
+    def handleErrorWith[A](fa: Id[A])(f: FetchError => Id[A]): Id[A] =
       try {
         fa
       } catch {
-        case ex: Throwable => f(ex)
+        case ex: FetchError => f(ex)
+      }
+    def raiseError[A](e: FetchError): Id[A] =
+      e match {
+        case FetchException(ex) => {
+            e.initCause(ex)
+            throw e
+          }
+        case other => throw other
       }
-    def raiseError[A](e: Throwable): Id[A]             = throw e
     def flatMap[A, B](fa: Id[A])(f: A => Id[B]): Id[B] = f(fa)
   }
 }

monix/shared/src/main/scala/monix.scala

@@ -45,10 +45,12 @@ object implicits {
     def pure[A](x: A): Task[A] =
       Task.now(x)
 
-    def handleErrorWith[A](fa: Task[A])(f: Throwable => Task[A]): Task[A] =
-      fa.onErrorHandleWith(f)
+    def handleErrorWith[A](fa: Task[A])(f: FetchError => Task[A]): Task[A] =
+      fa.onErrorHandleWith({
+        case e: FetchError => f(e)
+      })
 
-    def raiseError[A](e: Throwable): Task[A] =
+    def raiseError[A](e: FetchError): Task[A] =
       Task.raiseError(e)
 
     def flatMap[A, B](fa: Task[A])(f: A => Task[B]): Task[B] =

shared/src/main/scala/datasource.scala

@@ -29,7 +29,8 @@ trait DataSource[I, A] {
 
   /** The name of the data source.
     */
-  def name: DataSourceName = this.toString
+  def name: DataSourceName      = this.getClass.getName
+  override def toString: String = name
 
   /**
     * Derive a `DataSourceIdentity` from an identity, suitable for storing the result

shared/src/main/scala/env.scala

@@ -23,59 +23,36 @@ import scala.collection.immutable._
   * cache and the list of rounds that have been executed.
   */
 trait Env {
-  def cache: DataSourceCache
   def rounds: Seq[Round]
-
-  def cached: Seq[Round] =
-    rounds.filter(_.cached)
-
-  def uncached: Seq[Round] =
-    rounds.filterNot(_.cached)
-
-  def next(
-      newCache: DataSourceCache,
-      newRound: Round,
-      newIds: List[Any]
-  ): Env
+  def cache: DataSourceCache
+  def evolve(newRound: Round, newCache: DataSourceCache): Env
 }
 
 /**
   * A data structure that holds information about a fetch round.
   */
 case class Round(
     cache: DataSourceCache,
-    ds: DataSourceName,
-    kind: RoundKind,
-    startRound: Long,
-    endRound: Long,
-    cached: Boolean = false
+    request: FetchRequest,
+    response: Any,
+    start: Long,
+    end: Long
 ) {
-  def duration: Double = (endRound - startRound) / 1e6
-
-  def isConcurrent: Boolean = kind match {
-    case ConcurrentRound(_) => true
-    case _                  => false
-  }
+  def duration: Double = (end - start) / 1e6
 }
 
-sealed trait RoundKind
-final case class OneRound(id: Any)                            extends RoundKind
-final case class ManyRound(ids: List[Any])                    extends RoundKind
-final case class ConcurrentRound(ids: Map[String, List[Any]]) extends RoundKind
-
 /**
   * A concrete implementation of `Env` used in the default Fetch interpreter.
   */
 case class FetchEnv(
     cache: DataSourceCache,
-    ids: List[Any] = Nil,
     rounds: Queue[Round] = Queue.empty
 )
     extends Env {
-  def next(
-      newCache: DataSourceCache,
+
+  def evolve(
       newRound: Round,
-      newIds: List[Any]
+      newCache: DataSourceCache
   ): FetchEnv =
-    copy(cache = newCache, rounds = rounds :+ newRound, ids = newIds)
+    copy(rounds = rounds :+ newRound, cache = newCache)
 }