ecr caching

backend/src/main/scala/cromwell/backend/backend.scala

@@ -139,6 +139,8 @@ object CommonBackendConfigurationAttributes {
     "default-runtime-attributes.noAddress",
     "default-runtime-attributes.docker",
     "default-runtime-attributes.queueArn",
+    "default-runtime-attributes.awsBatchRetryAttempts",
+    "default-runtime-attributes.ulimits",
     "default-runtime-attributes.failOnStderr",
     "slow-job-warning-time",
     "dockerhub",

core/src/main/resources/reference.conf

@@ -418,6 +418,8 @@ docker {
     dockerhub.num-threads = 10
     quay.num-threads = 10
     alibabacloudcr.num-threads = 10
+    ecr.num-threads = 10
+    ecr-public.num-threads = 10
   }
 }
 

dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala

@@ -14,6 +14,7 @@ import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext}
 import cromwell.core.{Dispatcher, DockerConfiguration}
 import cromwell.docker.DockerInfoActor._
 import cromwell.docker.registryv2.DockerRegistryV2Abstract
+import cromwell.docker.registryv2.flows.aws.{AmazonEcr, AmazonEcrPublic}
 import cromwell.docker.registryv2.flows.alibabacloudcrregistry._
 import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry
 import cromwell.docker.registryv2.flows.google.GoogleRegistry
@@ -236,7 +237,9 @@ object DockerInfoActor {
       ("dockerhub", { c: DockerRegistryConfig => new DockerHubRegistry(c) }),
       ("google", { c: DockerRegistryConfig => new GoogleRegistry(c) }),
       ("quay", { c: DockerRegistryConfig => new QuayRegistry(c) }),
-      ("alibabacloudcr", {c: DockerRegistryConfig => new AlibabaCloudCRRegistry(c)})
+      ("alibabacloudcr", {c: DockerRegistryConfig => new AlibabaCloudCRRegistry(c)}),
+      ("ecr", {c: DockerRegistryConfig => new AmazonEcr(c)}),
+      ("ecr-public", {c: DockerRegistryConfig => new AmazonEcrPublic(c)})
     ).traverse[ErrorOr, DockerRegistry]({
       case (configPath, constructor) => DockerRegistryConfig.fromConfig(config.as[Config](configPath)).map(constructor)
     }).unsafe("Docker registry configuration")

dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2Abstract.scala

@@ -60,7 +60,7 @@ object DockerRegistryV2Abstract {
           )
       })
   }
-  
+
   // Placeholder exceptions that can be carried through IO before being converted to a DockerInfoFailedResponse
   private class Unauthorized() extends Exception
   private class NotFound() extends Exception
@@ -76,6 +76,8 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
   implicit val cs = IO.contextShift(ec)
   implicit val timer = IO.timer(ec)
 
+  protected val authorizationScheme: AuthScheme = AuthScheme.Bearer
+
   /**
     * This is the main function. Given a docker context and an http client, retrieve information about the docker image.
     */
@@ -204,7 +206,7 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
     * Request to get the manifest, using the auth token if provided
     */
   private def manifestRequest(token: Option[String], imageId: DockerImageIdentifier): IO[Request[IO]] = {
-    val authorizationHeader = token.map(t => Authorization(Credentials.Token(AuthScheme.Bearer, t)))
+    val authorizationHeader = token.map(t => Authorization(Credentials.Token(authorizationScheme, t)))
     val request = Method.GET(
       buildManifestUri(imageId),
       List(
@@ -235,13 +237,13 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
     * The response can be of 2 sorts:
     * - A manifest (https://docs.docker.com/registry/spec/manifest-v2-2/#image-manifest-field-descriptions)
     * - A manifest list which contains a list of pointers to other manifests (https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list)
-    * 
+    *
     * When a manifest list is returned, we need to pick one of the manifest pointers and make another request for that manifest.
-    * 
+    *
     * Because the different manifests in the list are (supposed to be) variations of the same image over different platforms,
     * we simply pick the first one here since we only care about the approximate size, and we don't expect it to change drastically
     * between platforms.
-    * If that assumption turns out to be incorrect, a smarter decision may need to be made to choose the manifest to lookup. 
+    * If that assumption turns out to be incorrect, a smarter decision may need to be made to choose the manifest to lookup.
     */
   private def parseManifest(dockerImageIdentifier: DockerImageIdentifier, token: Option[String])(response: Response[IO])(implicit client: Client[IO]): IO[Option[DockerManifest]] = response match {
     case Status.Successful(r) if r.headers.exists(_.value.equalsIgnoreCase(ManifestV2MediaType)) =>
@@ -268,14 +270,14 @@ abstract class DockerRegistryV2Abstract(override val config: DockerRegistryConfi
     }
   }
 
-  private def getDigestFromResponse(response: Response[IO]): IO[DockerHashResult] = response match {
+  protected def getDigestFromResponse(response: Response[IO]): IO[DockerHashResult] = response match {
     case Status.Successful(r) => extractDigestFromHeaders(r.headers)
     case Status.Unauthorized(_) => IO.raiseError(new Unauthorized)
     case Status.NotFound(_) => IO.raiseError(new NotFound)
     case failed => failed.as[String].flatMap(body => IO.raiseError(new Exception(s"Failed to get manifest: $body"))
     )
   }
-  
+
   private def extractDigestFromHeaders(headers: Headers) = {
     headers.find(a => a.toRaw.name.equals(DigestHeaderName)) match {
       case Some(digest) => IO.fromEither(DockerHashResult.fromString(digest.value).toEither)

dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AmazonEcr.scala

@@ -0,0 +1,42 @@
+package cromwell.docker.registryv2.flows.aws
+
+import cats.effect.IO
+import cromwell.docker.{DockerImageIdentifier, DockerInfoActor, DockerRegistryConfig}
+import org.http4s.AuthScheme
+import org.http4s.client.Client
+import software.amazon.awssdk.services.ecr.EcrClient
+
+import scala.compat.java8.OptionConverters._
+import scala.concurrent.Future
+
+class AmazonEcr(override val config: DockerRegistryConfig, ecrClient: EcrClient = EcrClient.create()) extends AmazonEcrAbstract(config) {
+
+  override protected val authorizationScheme: AuthScheme = AuthScheme.Basic
+
+  /**
+    * e.g 123456789012.dkr.ecr.us-east-1.amazonaws.com
+    */
+  override protected def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = {
+    var hostname = dockerImageIdentifier.hostAsString
+    if (hostname.lastIndexOf("/").equals(hostname.length -1)) {
+      hostname = hostname.substring(0, hostname.length -1)
+    }
+    hostname
+  }
+  /**
+    * Returns true if this flow is able to process this docker image,
+    * false otherwise
+    */
+  override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = dockerImageIdentifier.hostAsString.contains("amazonaws.com")
+
+  override protected def getToken(dockerInfoContext: DockerInfoActor.DockerInfoContext)(implicit client: Client[IO]): IO[Option[String]] = {
+    val eventualMaybeToken = Future(ecrClient.getAuthorizationToken
+      .authorizationData()
+      .stream()
+      .findFirst()
+      .asScala
+      .map(_.authorizationToken()))
+
+    IO.fromFuture(IO(eventualMaybeToken))
+  }
+}

dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AmazonEcrAbstract.scala

@@ -0,0 +1,41 @@
+package cromwell.docker.registryv2.flows.aws
+
+import cats.effect.IO
+import cromwell.docker.{DockerHashResult, DockerImageIdentifier, DockerInfoActor, DockerRegistryConfig}
+import cromwell.docker.registryv2.DockerRegistryV2Abstract
+import cromwell.docker.registryv2.flows.aws.EcrUtils.{EcrForbidden, EcrNotFound, EcrUnauthorized}
+import org.apache.commons.codec.digest.DigestUtils
+import org.http4s.{Header, Response, Status}
+
+abstract class AmazonEcrAbstract(override val config: DockerRegistryConfig) extends DockerRegistryV2Abstract(config) {
+
+  /**
+    * Not used as getToken is overridden
+    */
+  override protected def authorizationServerHostName(dockerImageIdentifier: DockerImageIdentifier): String = ""
+
+  /**
+    * Not used as getToken is overridden
+    */
+  override protected def buildTokenRequestHeaders(dockerInfoContext: DockerInfoActor.DockerInfoContext): List[Header] = List.empty
+
+  /**
+    * Amazon ECR repositories don't have a digest header in responses so we must made it from the manifest body
+    */
+  override protected def getDigestFromResponse(response: Response[IO]): IO[DockerHashResult] = response match {
+    case Status.Successful(r) => digestManifest(r.bodyText)
+    case Status.Unauthorized(_) => IO.raiseError(new EcrUnauthorized)
+    case Status.NotFound(_) => IO.raiseError(new EcrNotFound)
+    case Status.Forbidden(_) => IO.raiseError(new EcrForbidden)
+    case failed => failed.as[String].flatMap(body => IO.raiseError(new Exception(s"Failed to get manifest: $body")))
+  }
+
+  private def digestManifest(bodyText: fs2.Stream[IO, String]): IO[DockerHashResult] = {
+    bodyText
+      .compile
+      .string
+      .map(data => "sha256:"+DigestUtils.sha256Hex(data))
+      .map(DockerHashResult.fromString)
+      .flatMap(IO.fromTry)
+  }
+}

dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/AmazonEcrPublic.scala

@@ -0,0 +1,36 @@
+package cromwell.docker.registryv2.flows.aws
+
+import cats.effect.IO
+import cromwell.docker.{DockerImageIdentifier, DockerInfoActor, DockerRegistryConfig}
+import org.http4s.client.Client
+import software.amazon.awssdk.services.ecrpublic.EcrPublicClient
+import software.amazon.awssdk.services.ecrpublic.model.GetAuthorizationTokenRequest
+
+import scala.concurrent.Future
+
+
+class AmazonEcrPublic(override val config: DockerRegistryConfig, ecrClient: EcrPublicClient = EcrPublicClient.create()) extends AmazonEcrAbstract(config) {
+  /**
+    * public.ecr.aws
+    */
+  override protected def registryHostName(dockerImageIdentifier: DockerImageIdentifier): String = "public.ecr.aws"
+
+  /**
+    * Returns true if this flow is able to process this docker image,
+    * false otherwise
+    */
+  override def accepts(dockerImageIdentifier: DockerImageIdentifier): Boolean = dockerImageIdentifier.hostAsString.contains("public.ecr.aws")
+
+
+  override protected def getToken(dockerInfoContext: DockerInfoActor.DockerInfoContext)(implicit client: Client[IO]): IO[Option[String]] = {
+
+    val eventualMaybeToken: Future[Option[String]] = Future(
+      Option(ecrClient
+        .getAuthorizationToken(GetAuthorizationTokenRequest.builder().build())
+        .authorizationData.authorizationToken()
+      )
+    )
+
+    IO.fromFuture(IO(eventualMaybeToken))
+  }
+}

dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/aws/EcrUtils.scala

@@ -0,0 +1,9 @@
+package cromwell.docker.registryv2.flows.aws
+
+object EcrUtils {
+
+  case class EcrUnauthorized() extends Exception
+  case class EcrNotFound() extends Exception
+  case class EcrForbidden() extends Exception
+
+}

dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/aws/AmazonEcrPublicSpec.scala

@@ -0,0 +1,69 @@
+package cromwell.docker.registryv2.flows.aws
+
+import cats.effect.{IO, Resource}
+import cromwell.core.TestKitSuite
+import cromwell.docker.registryv2.DockerRegistryV2Abstract
+import cromwell.docker.{DockerImageIdentifier, DockerInfoActor, DockerInfoRequest, DockerRegistryConfig}
+import org.http4s.{Header, Headers, MediaType, Request, Response}
+import org.http4s.client.Client
+import org.http4s.headers.`Content-Type`
+import org.mockito.ArgumentMatchers.any
+import org.mockito.Mockito._
+import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
+import org.scalatest.flatspec.AnyFlatSpecLike
+import org.scalatest.matchers.should.Matchers
+import org.scalatestplus.mockito.MockitoSugar
+import software.amazon.awssdk.services.ecrpublic.model.{AuthorizationData, GetAuthorizationTokenRequest, GetAuthorizationTokenResponse}
+import software.amazon.awssdk.services.ecrpublic.EcrPublicClient
+
+class AmazonEcrPublicSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with MockitoSugar with BeforeAndAfter with PrivateMethodTester {
+  behavior of "AmazonEcrPublic"
+
+  val goodUri = "public.ecr.aws/amazonlinux/amazonlinux:latest"
+  val otherUri = "ubuntu:latest"
+
+
+  val mediaType: MediaType = MediaType.parse(DockerRegistryV2Abstract.ManifestV2MediaType).right.get
+  val contentType: Header = `Content-Type`(mediaType)
+  val mockEcrClient: EcrPublicClient = mock[EcrPublicClient]
+  implicit val mockIOClient: Client[IO] = Client({ _: Request[IO] =>
+    // This response will have an empty body, so we need to be explicit about the typing:
+    Resource.pure[IO, Response[IO]](Response(headers = Headers.of(contentType))) : Resource[IO, Response[IO]]
+  })
+
+  val registry = new AmazonEcrPublic(DockerRegistryConfig.default, mockEcrClient)
+
+  it should "Accept good URI" in {
+    val dockerImageIdentifier = DockerImageIdentifier.fromString(goodUri).get
+    registry.accepts(dockerImageIdentifier) shouldEqual true
+  }
+
+  it should "NOT accept other URI" in {
+    val dockerImageIdentifier = DockerImageIdentifier.fromString(otherUri).get
+    registry.accepts(dockerImageIdentifier) shouldEqual false
+  }
+
+  it should "have public.ecr.aws as registryHostName" in {
+    val registryHostNameMethod = PrivateMethod[String]('registryHostName)
+    registry invokePrivate registryHostNameMethod(DockerImageIdentifier.fromString(goodUri).get) shouldEqual "public.ecr.aws"
+  }
+
+  it should "return expected auth token" in {
+    val token = "auth-token"
+    val imageId = DockerImageIdentifier.fromString(goodUri).get
+    val dockerInfoRequest = DockerInfoRequest(imageId)
+    val context = DockerInfoActor.DockerInfoContext(request = dockerInfoRequest, replyTo =  emptyActor)
+
+    when(mockEcrClient.getAuthorizationToken(any[GetAuthorizationTokenRequest]()))
+      .thenReturn(GetAuthorizationTokenResponse
+        .builder()
+        .authorizationData(AuthorizationData
+          .builder()
+          .authorizationToken(token)
+          .build())
+        .build)
+
+    val getTokenMethod = PrivateMethod[IO[Option[String]]]('getToken)
+    registry invokePrivate getTokenMethod(context, mockIOClient) ensuring(io => io.unsafeRunSync().get == token)
+  }
+}

dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/aws/AmazonEcrSpec.scala

@@ -0,0 +1,72 @@
+package cromwell.docker.registryv2.flows.aws
+
+import cats.effect.{IO, Resource}
+import cromwell.core.TestKitSuite
+import cromwell.docker.registryv2.DockerRegistryV2Abstract
+import cromwell.docker.{DockerImageIdentifier, DockerInfoActor, DockerInfoRequest, DockerRegistryConfig}
+import org.http4s.{AuthScheme, Header, Headers, MediaType, Request, Response}
+import org.http4s.client.Client
+import org.http4s.headers.`Content-Type`
+import org.mockito.Mockito._
+import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
+import org.scalatest.flatspec.AnyFlatSpecLike
+import org.scalatest.matchers.should.Matchers
+import org.scalatestplus.mockito.MockitoSugar
+import software.amazon.awssdk.services.ecr.EcrClient
+import software.amazon.awssdk.services.ecr.model.{AuthorizationData, GetAuthorizationTokenResponse}
+
+class AmazonEcrSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with MockitoSugar with BeforeAndAfter with PrivateMethodTester{
+  behavior of "AmazonEcr"
+
+  val goodUri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/amazonlinux/amazonlinux:latest"
+  val otherUri = "ubuntu:latest"
+
+  val mediaType: MediaType = MediaType.parse(DockerRegistryV2Abstract.ManifestV2MediaType).right.get
+  val contentType: Header = `Content-Type`(mediaType)
+  val mockEcrClient: EcrClient = mock[EcrClient]
+  implicit val mockIOClient: Client[IO] = Client({ _: Request[IO] =>
+    // This response will have an empty body, so we need to be explicit about the typing:
+    Resource.pure[IO, Response[IO]](Response(headers = Headers.of(contentType))) : Resource[IO, Response[IO]]
+  })
+
+  val registry = new AmazonEcr(DockerRegistryConfig.default, mockEcrClient)
+
+  it should "accept good URI" in {
+    val dockerImageIdentifier = DockerImageIdentifier.fromString(goodUri).get
+    registry.accepts(dockerImageIdentifier) shouldEqual true
+  }
+
+  it should "NOT accept other URI" in {
+    val dockerImageIdentifier = DockerImageIdentifier.fromString(otherUri).get
+    registry.accepts(dockerImageIdentifier) shouldEqual false
+  }
+
+  it should "use Basic Auth Scheme" in {
+    val authSchemeMethod = PrivateMethod[AuthScheme]('authorizationScheme)
+    registry invokePrivate authSchemeMethod() shouldEqual AuthScheme.Basic
+  }
+
+  it should "return 123456789012.dkr.ecr.us-east-1.amazonaws.com as registryHostName" in {
+    val registryHostNameMethod = PrivateMethod[String]('registryHostName)
+    registry invokePrivate registryHostNameMethod(DockerImageIdentifier.fromString(goodUri).get) shouldEqual "123456789012.dkr.ecr.us-east-1.amazonaws.com"
+  }
+
+  it should "return expected auth token" in {
+    val token = "auth-token"
+    val imageId = DockerImageIdentifier.fromString(goodUri).get
+    val dockerInfoRequest = DockerInfoRequest(imageId)
+    val context = DockerInfoActor.DockerInfoContext(request = dockerInfoRequest, replyTo =  emptyActor)
+
+    when(mockEcrClient.getAuthorizationToken)
+      .thenReturn(GetAuthorizationTokenResponse
+        .builder()
+        .authorizationData(AuthorizationData
+          .builder()
+          .authorizationToken(token)
+          .build())
+        .build)
+
+    val getTokenMethod = PrivateMethod[IO[Option[String]]]('getToken)
+    registry invokePrivate getTokenMethod(context, mockIOClient) ensuring(io => io.unsafeRunSync().get == token)
+  }
+}

docs/LanguageSupport.md

@@ -26,7 +26,7 @@ As well as the changes to the WDL spec between draft-2 and 1.0, Cromwell also su
 ### CWL 1.0
 
 Cromwell provides support for Common Workflow Language (CWL), beginning with the core spec, and most heavily used requirements.
-If you spot a CWL feature that Cromwell doesn't support, please notify us using an issue on our github page!
+If you spot a CWL feature that Cromwell doesn't support, please notify us using an issue on our [Jira page](https://broadworkbench.atlassian.net/secure/RapidBoard.jspa?rapidView=39&view=planning.nodetail&issueLimit=100)!
 
 
 ## Future Language Support