Merge pull request #1565 from guardian/ag/feast-publication

Publication chain for Feast fronts

app/Components.scala

@@ -1,6 +1,7 @@
 import com.amazonaws.auth.AWSCredentialsProvider
 import software.amazon.awssdk.regions.{Region => WeirdRegion}
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+import com.amazonaws.services.sns.AmazonSNSClient
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, AwsCredentialsProvider, AwsCredentialsProviderChain, DefaultCredentialsProvider, ProfileCredentialsProvider}
 import conf.ApplicationConfiguration
 import config.{CustomGzipFilter, UpdateManager}
 import controllers._
@@ -20,13 +21,14 @@ import services._
 import services.editions.EditionsTemplating
 import services.editions.db.EditionsDB
 import services.editions.publishing.events.PublishEventsListener
-import services.editions.publishing.{EditionsBucket, EditionsPublishing}
+import services.editions.publishing.{EditionsBucket, FeastPublicationTarget, Publishing}
 import slices.{Containers, FixedContainers}
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient
 import thumbnails.ContainerThumbnails
 import tools.FaciaApiIO
 import updates.{BreakingNewsUpdate, StructuredLogger}
-import util.Acl
+import util.{Acl, TimestampGenerator}
+import services.editions.publishing.PublishedIssueFormatters._
 
 class AppComponents(context: Context, val config: ApplicationConfiguration)
  extends BaseFaciaControllerComponents(context) with EvolutionsComponents with DBComponents with HikariCPComponents {
@@ -54,14 +56,16 @@ class AppComponents(context: Context, val config: ApplicationConfiguration)
  .region(WeirdRegion.of(config.aws.region))
  .build()
  val s3Client = S3.client(oldAwsCredentials, config.aws.region)
+ val snsClient = AmazonSNSClient.builder().withCredentials(oldAwsCredentials).withRegion(config.aws.region).build()
  val acl = new Acl(permissions)
 
  // Editions services
  val editionsDb = new EditionsDB(config.postgres.url, config.postgres.user, config.postgres.password)
  val templating = new EditionsTemplating(EditionsAppTemplates.templates ++ FeastAppTemplates.templates, capi, ophan)
  val publishingBucket = new EditionsBucket(s3Client, config.aws.publishedEditionsIssuesBucket)
  val previewBucket = new EditionsBucket(s3Client, config.aws.previewEditionsIssuesBucket)
- val editionsPublishing = new EditionsPublishing(publishingBucket, previewBucket, editionsDb)
+ val feastPublicationTarget = new FeastPublicationTarget(snsClient, config, TimestampGenerator())
+ val editionsPublishing = new Publishing(publishingBucket, previewBucket, feastPublicationTarget, editionsDb)
  PublishEventsListener.apply(config, editionsDb).start
 
  // Controllers

app/conf/Configuration.scala

@@ -2,7 +2,6 @@ package conf
 
 import java.io.{File, FileInputStream, InputStream}
 import java.net.URL
-
 import com.amazonaws.AmazonClientException
 import com.amazonaws.auth._
 import com.amazonaws.auth.profile.ProfileCredentialsProvider
@@ -17,14 +16,16 @@ import com.amazonaws.services.rds.AmazonRDSClientBuilder
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder
 import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest
-import software.amazon.awssdk.auth.credentials.{AwsCredentialsProviderChain => NewAwsCredentialsProviderChain, ProfileCredentialsProvider => NewProfileCredentialsProvider, DefaultCredentialsProvider}
+import software.amazon.awssdk.auth.credentials.{DefaultCredentialsProvider, AwsCredentialsProviderChain => NewAwsCredentialsProviderChain, ProfileCredentialsProvider => NewProfileCredentialsProvider}
+
+import java.nio.charset.StandardCharsets
 
 class BadConfigurationException(msg: String) extends RuntimeException(msg)
 
 class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isProd: Boolean) extends Logging {
  private val propertiesFile = "/etc/gu/facia-tool.properties"
  private val installVars = new File(propertiesFile) match {
- case f if f.exists => IOUtils.toString(new FileInputStream(f))
+ case f if f.exists => IOUtils.toString(new FileInputStream(f), "UTF-8")
  case _ =>
  logger.warn("Missing configuration file $propertiesFile")
  ""
@@ -85,6 +86,8 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isP
  lazy val publishedEditionsIssuesBucket = getMandatoryString("aws.publishedEditionsIssuesBucket")
  lazy val previewEditionsIssuesBucket = getMandatoryString("aws.previewEditionsIssuesBucket")
 
+ lazy val feastAppPublicationTopic = getMandatoryString("feast_app.publication_topic")
+
  def cmsFrontsAccountCredentials: AWSCredentialsProvider = credentials.getOrElse(throw new BadConfigurationException("AWS credentials are not configured for CMS Fronts"))
  val credentials: Option[AWSCredentialsProvider] = {
  val provider = new AWSCredentialsProviderChain(
@@ -138,9 +141,9 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isP
  None
  }
  }
- val rdsClient = AmazonRDSClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
- val ssmClient = AWSSimpleSystemsManagementClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
- val s3Client = AmazonS3ClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
+ lazy val rdsClient = AmazonRDSClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
+ lazy val ssmClient = AWSSimpleSystemsManagementClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
+ lazy val s3Client = AmazonS3ClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build()
  }
 
  object postgres {

app/controllers/EditionsController.scala

@@ -10,13 +10,13 @@ import model.editions._
 import model.editions.templates.{CuratedPlatformDefinition, CuratedPlatformWithTemplate, EditionType}
 import model.forms._
 import net.logstash.logback.marker.Markers
-import play.api.libs.json.{JsObject, Json}
+import play.api.libs.json.{JsObject, JsValue, Json}
 import play.api.mvc.Result
 import services.Capi
 import services.editions.EditionsTemplating
 import services.editions.db.EditionsDB
 import services.editions.prefills.{CapiPrefillTimeParams, MetadataForLogging, PrefillParamsAdapter}
-import services.editions.publishing.EditionsPublishing
+import services.editions.publishing.Publishing
 import services.editions.publishing.PublishedIssueFormatters._
 import util.ContentUpgrade.rewriteBody
 import util.{SearchResponseUtil, UserUtil}
@@ -27,7 +27,7 @@ import scala.util.Try
 
 class EditionsController(db: EditionsDB,
  templating: EditionsTemplating,
- publishing: EditionsPublishing,
+ publishing: Publishing,
  capi: Capi,
  val deps: BaseFaciaControllerComponents)(implicit ec: ExecutionContext) extends BaseFaciaController(deps) with Logging {
 
@@ -86,7 +86,8 @@ class EditionsController(db: EditionsDB,
 
  def republishEditionsAppEditionsList = EditEditionsAuthAction { _ => {
  try {
- val raw = Json.toJson(Map("action" -> "editionList")).as[JsObject] + ("content", Json.toJson(getAvailableEditionsAppEditions))
+ //TODO: Make this a case class and serialise it properly
+ val raw = Json.toJson(Map("action" -> "editionList")).as[JsObject] + ("content", Json.toJson(EditionsAppTemplates.getAvailableEditionsAppTemplates))
  publishing.putEditionsList(raw.toString())
  Ok("Published. Please check processing has succeeded.")
  } catch {
@@ -189,15 +190,16 @@ class EditionsController(db: EditionsDB,
 
  def publishIssue(id: String, version: EditionIssueVersionId) = EditEditionsAuthAction { req =>
  val lastProofedIssueVersion = db.getLastProofedIssueVersion(id)
- // Protect against stale requests.
- if (lastProofedIssueVersion.filter(_.equals(version)).isEmpty) {
- BadRequest(s"Last proofed version of issue '${id}' is '${lastProofedIssueVersion.getOrElse("none")}', not '${version}'")
- } else {
- db.getIssue(id).map { issue =>
+
+ db.getIssue(id).map { issue =>
+ // Protect against stale requests, if our output platform supports proofing
+ if(issue.supportsProofing && !lastProofedIssueVersion.exists(_.equals(version))) {
+ BadRequest(s"Last proofed version of issue '${id}' is '${lastProofedIssueVersion.getOrElse("none")}', not '${version}'")
+ } else {
  publishing.publish(issue, req.user, version)
-  NoContent
- }.getOrElse(NotFound(s"Issue $id not found"))
- }
+ }
+ NoContent
+ }.getOrElse(NotFound(s"Issue $id not found"))
  }
 
  def getPrefillForCollection(id: String) = EditEditionsAuthAction { req =>
@@ -253,21 +255,9 @@ class EditionsController(db: EditionsDB,
  private def getAvailableCuratedPlatformEditions: Map[String, List[CuratedPlatformDefinition]] = {
  val feastAppEditions = FeastAppTemplates.getAvailableTemplates
 
- getAvailableEditionsAppEditions ++ Map(
+ EditionsAppTemplates.getAvailableEditionsAppTemplates ++ Map(
  "feastEditions" -> feastAppEditions,
  )
  }
 
- private def getAvailableEditionsAppEditions: Map[String, List[CuratedPlatformDefinition]] = {
- val allEditions = EditionsAppTemplates.getAvailableTemplates
- val regionalEditions = allEditions.filter(e => e.editionType == EditionType.Regional)
- val specialEditions = allEditions.filter(e => e.editionType == EditionType.Special)
- val trainingEditions = allEditions.filter(e => e.editionType == EditionType.Training)
-
- Map(
- "regionalEditions" -> regionalEditions,
- "specialEditions" -> specialEditions,
- "trainingEditions" -> trainingEditions,
- )
- }
 }

app/model/FeastAppModel.scala

@@ -0,0 +1,47 @@
+package model
+
+import model.editions.Edition
+import play.api.libs.json._
+
+import java.time.LocalDate
+
+object FeastAppModel {
+ sealed trait ContainerItem
+
+ case class RecipeIdentifier(id:String)
+ case class Recipe(recipe:RecipeIdentifier) extends ContainerItem
+ case class Chef(backgroundHex:Option[String], id:String, image:Option[String], bio: String, foregroundHex:Option[String]) extends ContainerItem
+ case class Palette(backgroundHex:String, foregroundHex:String)
+ case class SubCollection(byline:Option[String], darkPalette:Option[Palette], image:Option[String], body:Option[String], title:String, lightPalette:Option[Palette], recipes:Seq[String]) extends ContainerItem
+
+ case class FeastAppContainer(id:String, title:String, body:Option[String], items:Seq[ContainerItem])
+ //type FeastAppCuration = Map[String, IndexedSeq[FeastAppContainer]]
+
+ case class FeastAppCuration(
+ id:String,
+ edition:Edition,
+ issueDate:LocalDate,
+ version:String,
+ fronts:Map[String,IndexedSeq[FeastAppContainer]]
+ )
+
+ implicit val recipeIdentifierFormat:Format[RecipeIdentifier] = Json.format[RecipeIdentifier]
+ implicit val recipeFormat:Format[Recipe] = Json.format[Recipe]
+ implicit val chefFormat:Format[Chef] = Json.format[Chef]
+ implicit val paletteFormat:Format[Palette] = Json.format[Palette]
+ implicit val subCollectionFormat:Format[SubCollection] = Json.format[SubCollection]
+
+ implicit val containerItemFormat:Format[ContainerItem] = Format.apply(
+ jsValue=> {
+ recipeFormat.reads(jsValue) orElse chefFormat.reads(jsValue) orElse subCollectionFormat.reads(jsValue)
+ },
+ {
+ case o:Recipe=>recipeFormat.writes(o)
+ case o:Chef=>chefFormat.writes(o)
+ case o:SubCollection=>subCollectionFormat.writes(o)
+ }
+ )
+ implicit val feastAppContainerFormat:Format[FeastAppContainer] = Json.format[FeastAppContainer]
+ implicit val feastAppCurationFormat:Format[FeastAppCuration] = Json.format[FeastAppCuration]
+
+}

app/model/editions/EditionsAppTemplates.scala

@@ -30,6 +30,24 @@ object EditionsAppTemplates {
  )
 
  val getAvailableTemplates: List[EditionsAppDefinitionWithTemplate] = templates.values.toList
+
+ /**
+ * Returns available Editons app templates as a Map which differentiates the various classes
+ * of edition. This is used for grouping purposes in both the UI and the publication logic
+ * @return a Map relating the edition class to a list of the relevant types.
+ */
+ def getAvailableEditionsAppTemplates: Map[String, List[CuratedPlatformDefinition]] = {
+ val allEditions = getAvailableTemplates
+ val regionalEditions = allEditions.filter(e => e.editionType == EditionType.Regional)
+ val specialEditions = allEditions.filter(e => e.editionType == EditionType.Special)
+ val trainingEditions = allEditions.filter(e => e.editionType == EditionType.Training)
+
+ Map(
+ "regionalEditions" -> regionalEditions,
+ "specialEditions" -> specialEditions,
+ "trainingEditions" -> trainingEditions,
+ )
+ }
 }
 
 object FeastAppTemplates {
@@ -114,15 +132,6 @@ object Edition extends PlayEnum[Edition] {
  override def values = findValues
 }
 
-sealed abstract class FeastEditions extends EnumEntry with Hyphencase
-
-object FeastEditions extends PlayEnum[FeastEditions] {
- case object FeastNorthernHemisphere extends FeastEditions
- case object FeastSouthernHemisphere extends FeastEditions
-
- override def values = findValues
-}
-
 case class FrontPresentation(swatch: Swatch) {
  implicit def frontPresentationFormat = Json.format[FrontPresentation]
 }

app/model/editions/EditionsIssue.scala

@@ -29,7 +29,8 @@ case class EditionsIssue(
  launchedOn: Option[Long],
  launchedBy: Option[String],
  launchedEmail: Option[String],
- fronts: List[EditionsFront]
+ fronts: List[EditionsFront],
+ supportsProofing: Boolean,
 ) {
 
  // This is a no-op placeholder which matches the UK Daily Edition value.
@@ -55,17 +56,18 @@ case class EditionsIssue(
  .map(_.toPublishedFront) // convert
  .filterNot(_.collections.isEmpty), // drop fronts that contain no collections
  EditionsAppTemplates.templates.get(edition).map(_.notificationUTCOffset).getOrElse(defaultOffset),
- EditionsAppTemplates.templates.get(edition).map(_.topic)
+ EditionsAppTemplates.templates.get(edition).map(_.topic),
  )
 }
 
 object EditionsIssue {
  implicit val writes: OWrites[EditionsIssue] = Json.writes[EditionsIssue]
 
  def fromRow(rs: WrappedResultSet, prefix: String = ""): EditionsIssue = {
+ val edition = Edition.withName(rs.string(prefix + "name"))
  EditionsIssue(
  rs.string(prefix + "id"),
- Edition.withName(rs.string(prefix + "name")),
+ edition,
  rs.string(prefix + "timezone_id"),
  rs.localDate(prefix + "issue_date"),
  rs.zonedDateTime(prefix + "created_on").toInstant.toEpochMilli,
@@ -74,7 +76,8 @@ object EditionsIssue {
  rs.zonedDateTimeOpt(prefix + "launched_on").map(_.toInstant.toEpochMilli),
  rs.stringOpt(prefix + "launched_by"),
  rs.stringOpt(prefix + "launched_email"),
- Nil
+ Nil,
+ supportsProofing = EditionsAppTemplates.templates.contains(edition) //proofing is supported by Editions but not Feast
  )
  }
 }

app/model/editions/templates/feast/FeastNorthernHemisphere.scala

@@ -25,7 +25,7 @@ object FeastNorthernHemisphere extends FeastAppEdition {
  )
 
  val MeatFreeFront: FrontTemplate = front(
- "Meat-Free Recipes",
+ "Meat-Free",
  collection("Dish of the day"),
  collection("Collection 2"),
  collection("Collection 3"),

app/model/editions/templates/feast/FeastSouthernHemisphere.scala

@@ -13,7 +13,7 @@ object FeastSouthernHemisphere extends FeastAppEdition {
  override val notificationUTCOffset = 0
 
  val MainFront: FrontTemplate = front(
- "Southern hemisphere",
+ "All Recipes",
  collection("Dish of the day"),
  collection("Collection 2"),
  collection("Collection 3"),
@@ -26,7 +26,7 @@ object FeastSouthernHemisphere extends FeastAppEdition {
  )
 
  val MeatFreeFront: FrontTemplate = front(
- "Southern hemisphere",
+ "Meat-Free",
  collection("Dish of the day"),
  collection("Collection 2"),
  collection("Collection 3"),

app/services/editions/db/IssueQueries.scala

@@ -372,7 +372,7 @@ trait IssueQueries extends Logging {
  """.execute().apply()
  } match {
  case Success(_) => {
- logger.info("successfully inserted issue version event message:${event.message}")(event.toLogMarker)
+ logger.info(s"successfully inserted issue version event message:${event.message}")(event.toLogMarker)
  true
  }
  case Failure(exception: PSQLException) if exception.getSQLState == ForeignKeyViolationSQLState => {