diff --git a/app/Components.scala b/app/Components.scala index 6f21b626381..968f4797ea9 100644 --- a/app/Components.scala +++ b/app/Components.scala @@ -1,6 +1,7 @@ import com.amazonaws.auth.AWSCredentialsProvider import software.amazon.awssdk.regions.{Region => WeirdRegion} -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import com.amazonaws.services.sns.AmazonSNSClient +import software.amazon.awssdk.auth.credentials.{AwsCredentials, AwsCredentialsProvider, AwsCredentialsProviderChain, DefaultCredentialsProvider, ProfileCredentialsProvider} import conf.ApplicationConfiguration import config.{CustomGzipFilter, UpdateManager} import controllers._ @@ -20,13 +21,14 @@ import services._ import services.editions.EditionsTemplating import services.editions.db.EditionsDB import services.editions.publishing.events.PublishEventsListener -import services.editions.publishing.{EditionsBucket, EditionsPublishing} +import services.editions.publishing.{EditionsBucket, FeastPublicationTarget, Publishing} import slices.{Containers, FixedContainers} import software.amazon.awssdk.services.dynamodb.DynamoDbClient import thumbnails.ContainerThumbnails import tools.FaciaApiIO import updates.{BreakingNewsUpdate, StructuredLogger} -import util.Acl +import util.{Acl, TimestampGenerator} +import services.editions.publishing.PublishedIssueFormatters._ class AppComponents(context: Context, val config: ApplicationConfiguration) extends BaseFaciaControllerComponents(context) with EvolutionsComponents with DBComponents with HikariCPComponents { @@ -54,6 +56,7 @@ class AppComponents(context: Context, val config: ApplicationConfiguration) .region(WeirdRegion.of(config.aws.region)) .build() val s3Client = S3.client(oldAwsCredentials, config.aws.region) + val snsClient = AmazonSNSClient.builder().withCredentials(oldAwsCredentials).withRegion(config.aws.region).build() val acl = new Acl(permissions) // Editions services @@ -61,7 +64,8 @@ class AppComponents(context: Context, val config: ApplicationConfiguration) val templating = new EditionsTemplating(EditionsAppTemplates.templates ++ FeastAppTemplates.templates, capi, ophan) val publishingBucket = new EditionsBucket(s3Client, config.aws.publishedEditionsIssuesBucket) val previewBucket = new EditionsBucket(s3Client, config.aws.previewEditionsIssuesBucket) - val editionsPublishing = new EditionsPublishing(publishingBucket, previewBucket, editionsDb) + val feastPublicationTarget = new FeastPublicationTarget(snsClient, config, TimestampGenerator()) + val editionsPublishing = new Publishing(publishingBucket, previewBucket, feastPublicationTarget, editionsDb) PublishEventsListener.apply(config, editionsDb).start // Controllers diff --git a/app/conf/Configuration.scala b/app/conf/Configuration.scala index e0c4d8fa84c..419a21050be 100644 --- a/app/conf/Configuration.scala +++ b/app/conf/Configuration.scala @@ -2,7 +2,6 @@ package conf import java.io.{File, FileInputStream, InputStream} import java.net.URL - import com.amazonaws.AmazonClientException import com.amazonaws.auth._ import com.amazonaws.auth.profile.ProfileCredentialsProvider @@ -17,14 +16,16 @@ import com.amazonaws.services.rds.AmazonRDSClientBuilder import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest -import software.amazon.awssdk.auth.credentials.{AwsCredentialsProviderChain => NewAwsCredentialsProviderChain, ProfileCredentialsProvider => NewProfileCredentialsProvider, DefaultCredentialsProvider} +import software.amazon.awssdk.auth.credentials.{DefaultCredentialsProvider, AwsCredentialsProviderChain => NewAwsCredentialsProviderChain, ProfileCredentialsProvider => NewProfileCredentialsProvider} + +import java.nio.charset.StandardCharsets class BadConfigurationException(msg: String) extends RuntimeException(msg) class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isProd: Boolean) extends Logging { private val propertiesFile = "/etc/gu/facia-tool.properties" private val installVars = new File(propertiesFile) match { - case f if f.exists => IOUtils.toString(new FileInputStream(f)) + case f if f.exists => IOUtils.toString(new FileInputStream(f), "UTF-8") case _ => logger.warn("Missing configuration file $propertiesFile") "" @@ -85,6 +86,8 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isP lazy val publishedEditionsIssuesBucket = getMandatoryString("aws.publishedEditionsIssuesBucket") lazy val previewEditionsIssuesBucket = getMandatoryString("aws.previewEditionsIssuesBucket") + lazy val feastAppPublicationTopic = getMandatoryString("feast_app.publication_topic") + def cmsFrontsAccountCredentials: AWSCredentialsProvider = credentials.getOrElse(throw new BadConfigurationException("AWS credentials are not configured for CMS Fronts")) val credentials: Option[AWSCredentialsProvider] = { val provider = new AWSCredentialsProviderChain( @@ -138,9 +141,9 @@ class ApplicationConfiguration(val playConfiguration: PlayConfiguration, val isP None } } - val rdsClient = AmazonRDSClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() - val ssmClient = AWSSimpleSystemsManagementClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() - val s3Client = AmazonS3ClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() + lazy val rdsClient = AmazonRDSClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() + lazy val ssmClient = AWSSimpleSystemsManagementClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() + lazy val s3Client = AmazonS3ClientBuilder.standard().withCredentials(cmsFrontsAccountCredentials).withRegion(region).build() } object postgres { diff --git a/app/controllers/EditionsController.scala b/app/controllers/EditionsController.scala index 26124b891fd..74f454197f3 100644 --- a/app/controllers/EditionsController.scala +++ b/app/controllers/EditionsController.scala @@ -10,13 +10,13 @@ import model.editions._ import model.editions.templates.{CuratedPlatformDefinition, CuratedPlatformWithTemplate, EditionType} import model.forms._ import net.logstash.logback.marker.Markers -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsObject, JsValue, Json} import play.api.mvc.Result import services.Capi import services.editions.EditionsTemplating import services.editions.db.EditionsDB import services.editions.prefills.{CapiPrefillTimeParams, MetadataForLogging, PrefillParamsAdapter} -import services.editions.publishing.EditionsPublishing +import services.editions.publishing.Publishing import services.editions.publishing.PublishedIssueFormatters._ import util.ContentUpgrade.rewriteBody import util.{SearchResponseUtil, UserUtil} @@ -27,7 +27,7 @@ import scala.util.Try class EditionsController(db: EditionsDB, templating: EditionsTemplating, - publishing: EditionsPublishing, + publishing: Publishing, capi: Capi, val deps: BaseFaciaControllerComponents)(implicit ec: ExecutionContext) extends BaseFaciaController(deps) with Logging { @@ -86,7 +86,8 @@ class EditionsController(db: EditionsDB, def republishEditionsAppEditionsList = EditEditionsAuthAction { _ => { try { - val raw = Json.toJson(Map("action" -> "editionList")).as[JsObject] + ("content", Json.toJson(getAvailableEditionsAppEditions)) + //TODO: Make this a case class and serialise it properly + val raw = Json.toJson(Map("action" -> "editionList")).as[JsObject] + ("content", Json.toJson(EditionsAppTemplates.getAvailableEditionsAppTemplates)) publishing.putEditionsList(raw.toString()) Ok("Published. Please check processing has succeeded.") } catch { @@ -189,15 +190,16 @@ class EditionsController(db: EditionsDB, def publishIssue(id: String, version: EditionIssueVersionId) = EditEditionsAuthAction { req => val lastProofedIssueVersion = db.getLastProofedIssueVersion(id) - // Protect against stale requests. - if (lastProofedIssueVersion.filter(_.equals(version)).isEmpty) { - BadRequest(s"Last proofed version of issue '${id}' is '${lastProofedIssueVersion.getOrElse("none")}', not '${version}'") - } else { - db.getIssue(id).map { issue => + + db.getIssue(id).map { issue => + // Protect against stale requests, if our output platform supports proofing + if(issue.supportsProofing && !lastProofedIssueVersion.exists(_.equals(version))) { + BadRequest(s"Last proofed version of issue '${id}' is '${lastProofedIssueVersion.getOrElse("none")}', not '${version}'") + } else { publishing.publish(issue, req.user, version) - NoContent - }.getOrElse(NotFound(s"Issue $id not found")) - } + } + NoContent + }.getOrElse(NotFound(s"Issue $id not found")) } def getPrefillForCollection(id: String) = EditEditionsAuthAction { req => @@ -253,21 +255,9 @@ class EditionsController(db: EditionsDB, private def getAvailableCuratedPlatformEditions: Map[String, List[CuratedPlatformDefinition]] = { val feastAppEditions = FeastAppTemplates.getAvailableTemplates - getAvailableEditionsAppEditions ++ Map( + EditionsAppTemplates.getAvailableEditionsAppTemplates ++ Map( "feastEditions" -> feastAppEditions, ) } - private def getAvailableEditionsAppEditions: Map[String, List[CuratedPlatformDefinition]] = { - val allEditions = EditionsAppTemplates.getAvailableTemplates - val regionalEditions = allEditions.filter(e => e.editionType == EditionType.Regional) - val specialEditions = allEditions.filter(e => e.editionType == EditionType.Special) - val trainingEditions = allEditions.filter(e => e.editionType == EditionType.Training) - - Map( - "regionalEditions" -> regionalEditions, - "specialEditions" -> specialEditions, - "trainingEditions" -> trainingEditions, - ) - } } diff --git a/app/model/FeastAppModel.scala b/app/model/FeastAppModel.scala new file mode 100644 index 00000000000..3dc4492d93b --- /dev/null +++ b/app/model/FeastAppModel.scala @@ -0,0 +1,47 @@ +package model + +import model.editions.Edition +import play.api.libs.json._ + +import java.time.LocalDate + +object FeastAppModel { + sealed trait ContainerItem + + case class RecipeIdentifier(id:String) + case class Recipe(recipe:RecipeIdentifier) extends ContainerItem + case class Chef(backgroundHex:Option[String], id:String, image:Option[String], bio: String, foregroundHex:Option[String]) extends ContainerItem + case class Palette(backgroundHex:String, foregroundHex:String) + case class SubCollection(byline:Option[String], darkPalette:Option[Palette], image:Option[String], body:Option[String], title:String, lightPalette:Option[Palette], recipes:Seq[String]) extends ContainerItem + + case class FeastAppContainer(id:String, title:String, body:Option[String], items:Seq[ContainerItem]) + //type FeastAppCuration = Map[String, IndexedSeq[FeastAppContainer]] + + case class FeastAppCuration( + id:String, + edition:Edition, + issueDate:LocalDate, + version:String, + fronts:Map[String,IndexedSeq[FeastAppContainer]] + ) + + implicit val recipeIdentifierFormat:Format[RecipeIdentifier] = Json.format[RecipeIdentifier] + implicit val recipeFormat:Format[Recipe] = Json.format[Recipe] + implicit val chefFormat:Format[Chef] = Json.format[Chef] + implicit val paletteFormat:Format[Palette] = Json.format[Palette] + implicit val subCollectionFormat:Format[SubCollection] = Json.format[SubCollection] + + implicit val containerItemFormat:Format[ContainerItem] = Format.apply( + jsValue=> { + recipeFormat.reads(jsValue) orElse chefFormat.reads(jsValue) orElse subCollectionFormat.reads(jsValue) + }, + { + case o:Recipe=>recipeFormat.writes(o) + case o:Chef=>chefFormat.writes(o) + case o:SubCollection=>subCollectionFormat.writes(o) + } + ) + implicit val feastAppContainerFormat:Format[FeastAppContainer] = Json.format[FeastAppContainer] + implicit val feastAppCurationFormat:Format[FeastAppCuration] = Json.format[FeastAppCuration] + +} diff --git a/app/model/editions/EditionsAppTemplates.scala b/app/model/editions/EditionsAppTemplates.scala index eba57280555..abf76b8b7f0 100644 --- a/app/model/editions/EditionsAppTemplates.scala +++ b/app/model/editions/EditionsAppTemplates.scala @@ -30,6 +30,24 @@ object EditionsAppTemplates { ) val getAvailableTemplates: List[EditionsAppDefinitionWithTemplate] = templates.values.toList + + /** + * Returns available Editons app templates as a Map which differentiates the various classes + * of edition. This is used for grouping purposes in both the UI and the publication logic + * @return a Map relating the edition class to a list of the relevant types. + */ + def getAvailableEditionsAppTemplates: Map[String, List[CuratedPlatformDefinition]] = { + val allEditions = getAvailableTemplates + val regionalEditions = allEditions.filter(e => e.editionType == EditionType.Regional) + val specialEditions = allEditions.filter(e => e.editionType == EditionType.Special) + val trainingEditions = allEditions.filter(e => e.editionType == EditionType.Training) + + Map( + "regionalEditions" -> regionalEditions, + "specialEditions" -> specialEditions, + "trainingEditions" -> trainingEditions, + ) + } } object FeastAppTemplates { @@ -114,15 +132,6 @@ object Edition extends PlayEnum[Edition] { override def values = findValues } -sealed abstract class FeastEditions extends EnumEntry with Hyphencase - -object FeastEditions extends PlayEnum[FeastEditions] { - case object FeastNorthernHemisphere extends FeastEditions - case object FeastSouthernHemisphere extends FeastEditions - - override def values = findValues -} - case class FrontPresentation(swatch: Swatch) { implicit def frontPresentationFormat = Json.format[FrontPresentation] } diff --git a/app/model/editions/EditionsIssue.scala b/app/model/editions/EditionsIssue.scala index 48b36370a80..003a8eeb065 100644 --- a/app/model/editions/EditionsIssue.scala +++ b/app/model/editions/EditionsIssue.scala @@ -29,7 +29,8 @@ case class EditionsIssue( launchedOn: Option[Long], launchedBy: Option[String], launchedEmail: Option[String], - fronts: List[EditionsFront] + fronts: List[EditionsFront], + supportsProofing: Boolean, ) { // This is a no-op placeholder which matches the UK Daily Edition value. @@ -55,7 +56,7 @@ case class EditionsIssue( .map(_.toPublishedFront) // convert .filterNot(_.collections.isEmpty), // drop fronts that contain no collections EditionsAppTemplates.templates.get(edition).map(_.notificationUTCOffset).getOrElse(defaultOffset), - EditionsAppTemplates.templates.get(edition).map(_.topic) + EditionsAppTemplates.templates.get(edition).map(_.topic), ) } @@ -63,9 +64,10 @@ object EditionsIssue { implicit val writes: OWrites[EditionsIssue] = Json.writes[EditionsIssue] def fromRow(rs: WrappedResultSet, prefix: String = ""): EditionsIssue = { + val edition = Edition.withName(rs.string(prefix + "name")) EditionsIssue( rs.string(prefix + "id"), - Edition.withName(rs.string(prefix + "name")), + edition, rs.string(prefix + "timezone_id"), rs.localDate(prefix + "issue_date"), rs.zonedDateTime(prefix + "created_on").toInstant.toEpochMilli, @@ -74,7 +76,8 @@ object EditionsIssue { rs.zonedDateTimeOpt(prefix + "launched_on").map(_.toInstant.toEpochMilli), rs.stringOpt(prefix + "launched_by"), rs.stringOpt(prefix + "launched_email"), - Nil + Nil, + supportsProofing = EditionsAppTemplates.templates.contains(edition) //proofing is supported by Editions but not Feast ) } } diff --git a/app/model/editions/templates/feast/FeastNorthernHemisphere.scala b/app/model/editions/templates/feast/FeastNorthernHemisphere.scala index 0574fdf6d1b..3185084736a 100644 --- a/app/model/editions/templates/feast/FeastNorthernHemisphere.scala +++ b/app/model/editions/templates/feast/FeastNorthernHemisphere.scala @@ -25,7 +25,7 @@ object FeastNorthernHemisphere extends FeastAppEdition { ) val MeatFreeFront: FrontTemplate = front( - "Meat-Free Recipes", + "Meat-Free", collection("Dish of the day"), collection("Collection 2"), collection("Collection 3"), diff --git a/app/model/editions/templates/feast/FeastSouthernHemisphere.scala b/app/model/editions/templates/feast/FeastSouthernHemisphere.scala index 05f167da4bd..608aec872e4 100644 --- a/app/model/editions/templates/feast/FeastSouthernHemisphere.scala +++ b/app/model/editions/templates/feast/FeastSouthernHemisphere.scala @@ -13,7 +13,7 @@ object FeastSouthernHemisphere extends FeastAppEdition { override val notificationUTCOffset = 0 val MainFront: FrontTemplate = front( - "Southern hemisphere", + "All Recipes", collection("Dish of the day"), collection("Collection 2"), collection("Collection 3"), @@ -26,7 +26,7 @@ object FeastSouthernHemisphere extends FeastAppEdition { ) val MeatFreeFront: FrontTemplate = front( - "Southern hemisphere", + "Meat-Free", collection("Dish of the day"), collection("Collection 2"), collection("Collection 3"), diff --git a/app/services/editions/db/IssueQueries.scala b/app/services/editions/db/IssueQueries.scala index 5ce35fe38ea..1d9d8e609ae 100644 --- a/app/services/editions/db/IssueQueries.scala +++ b/app/services/editions/db/IssueQueries.scala @@ -372,7 +372,7 @@ trait IssueQueries extends Logging { """.execute().apply() } match { case Success(_) => { - logger.info("successfully inserted issue version event message:${event.message}")(event.toLogMarker) + logger.info(s"successfully inserted issue version event message:${event.message}")(event.toLogMarker) true } case Failure(exception: PSQLException) if exception.getSQLState == ForeignKeyViolationSQLState => { diff --git a/app/services/editions/publishing/EditionsBucket.scala b/app/services/editions/publishing/EditionsBucket.scala index c3b9d6303f9..1b4ca9300d0 100644 --- a/app/services/editions/publishing/EditionsBucket.scala +++ b/app/services/editions/publishing/EditionsBucket.scala @@ -4,37 +4,56 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest, PutObjectResult} import com.amazonaws.util.StringInputStream import model.editions.PublishableIssue -import play.api.libs.json.Json +import play.api.libs.json.{Json, Writes} import PublishedIssueFormatters._ +import com.typesafe.scalalogging.LazyLogging +import org.apache.commons.lang3.builder.{ReflectionToStringBuilder, ToStringStyle} + +import java.nio.charset.StandardCharsets + +object EditionsBucket extends LazyLogging { -object EditionsBucket { def createIssuePrefix(issue: PublishableIssue): String = s"${issue.name.entryName}/${issue.issueDate.toString}" def createIssueFilename(issue: PublishableIssue): String = s"${issue.version}.json" def createKey(issue: PublishableIssue): String = s"${createIssuePrefix(issue)}/${createIssueFilename(issue)}" - val objectMetadata: ObjectMetadata = { + val baseMetadata: ObjectMetadata = { val metadata = new ObjectMetadata() metadata.setContentType("application/json") metadata } - def createPutObjectRequest(bucketName: String, issue: PublishableIssue): PutObjectRequest = { - val key = EditionsBucket.createKey(issue) + def createPutObjectRequest[T:Writes](bucketName: String, key: String, issue: T): PutObjectRequest = { val issueJson = Json.stringify(Json.toJson(issue)) - new PutObjectRequest(bucketName, key, new StringInputStream(issueJson), EditionsBucket.objectMetadata) + + //Why do we do this? Well, because if we are sending a streaming PutObjectRequest then S3 requires the length of the stream + //If it's not explicitly set in ObjectMetadata, then the AWS SDK will consume the entire string into memory just to measure the length. + //This is pointless as we already _have_ it in memory here (and it creates un-necessary log noise). So we need to put the BYTE length into the header. + //Note that the byte length of a UTF-8 string can easily be greater than the character length, which is why we need to use getBytes here. + val metadata = baseMetadata + metadata.setContentLength(issueJson.getBytes(StandardCharsets.UTF_8).length) + new PutObjectRequest(bucketName, key, new StringInputStream(issueJson), metadata) } } -class EditionsBucket(s3Client: AmazonS3, bucketName: String) { - def putIssue(issue: PublishableIssue): PutObjectResult = { - val request = EditionsBucket.createPutObjectRequest(bucketName, issue) +class EditionsBucket(s3Client: AmazonS3, bucketName: String) extends PublicationTarget with LazyLogging { + override def putIssue(issue: PublishableIssue, key: Option[String]=None): Unit = { + val outputKey = key.getOrElse(EditionsBucket.createKey(issue)) + putIssueJson(issue, outputKey) + } + + override def putIssueJson[T: Writes](content: T, key:String): Unit = { + val request = EditionsBucket.createPutObjectRequest(bucketName, key, content) + logger.info(ReflectionToStringBuilder.toString(request, ToStringStyle.MULTI_LINE_STYLE)) s3Client.putObject(request) } - def putEditionsList(rawJson: String) = { - val request = new PutObjectRequest(bucketName, "editionsList", new StringInputStream(rawJson), EditionsBucket.objectMetadata) + def putEditionsList(rawJson: String): Unit = { + val metadata = EditionsBucket.baseMetadata + metadata.setContentLength(rawJson.getBytes(StandardCharsets.UTF_8).length) + val request = new PutObjectRequest(bucketName, "editionsList", new StringInputStream(rawJson), metadata) s3Client.putObject(request) } } diff --git a/app/services/editions/publishing/EditionsPublishing.scala b/app/services/editions/publishing/EditionsPublishing.scala deleted file mode 100644 index a9363805655..00000000000 --- a/app/services/editions/publishing/EditionsPublishing.scala +++ /dev/null @@ -1,70 +0,0 @@ -package services.editions.publishing - -import java.time.OffsetDateTime - -import com.gu.pandomainauth.model.User -import logging.Logging -import model.editions.{EditionsIssue, PublishAction} -import net.logstash.logback.marker.Markers -import services.editions.db.EditionsDB - -import scala.jdk.CollectionConverters._ - -class EditionsPublishing(publishedBucket: EditionsBucket, previewBucket: EditionsBucket, db: EditionsDB) extends Logging { - - def updatePreview(issue: EditionsIssue) = { - val previewIssue = issue.toPreviewIssue - previewBucket.putIssue(previewIssue) - } - - def proof(issue: EditionsIssue, user: User, now: OffsetDateTime) = { - - val action = PublishAction.proof - - val versionId = db.createIssueVersion(issue.id, user, now) - - val markers = Markers.appendEntries( - Map( - "issue-action" -> action.toString, - "issue-id" -> issue.id, - "issue-date" -> issue.issueDate.toString, - "version" -> versionId, - "user" -> user.email - ).asJava - ) - - logger.info(s"Uploading $action request for issue ${issue.id} to S3")(markers) - - val publishedIssue = issue.toPublishableIssue(versionId, action) - - // Archive a copy - publishedBucket.putIssue(publishedIssue) - } - - def putEditionsList(rawJson: String) = { - publishedBucket.putEditionsList(rawJson) - } - - def publish(issue: EditionsIssue, user: User, version: String) = { - - val action = PublishAction.proof - - val markers = Markers.appendEntries( - Map( - "issue-action" -> action.toString, - "issue-id" -> issue.id, - "issue-date" -> issue.issueDate.toString, - "version" -> version, - "user" -> user.email - ).asJava - ) - - logger.info(s"Uploading $action request for issue ${issue.id} to S3")(markers) - - val publishedIssue = issue.toPublishableIssue(version, PublishAction.publish) - - // Archive a copy - publishedBucket.putIssue(publishedIssue) - - } -} diff --git a/app/services/editions/publishing/FeastPublicationTarget.scala b/app/services/editions/publishing/FeastPublicationTarget.scala new file mode 100644 index 00000000000..00657de9014 --- /dev/null +++ b/app/services/editions/publishing/FeastPublicationTarget.scala @@ -0,0 +1,76 @@ +package services.editions.publishing +import com.amazonaws.services.sns.AmazonSNS +import com.amazonaws.services.sns.model.{MessageAttributeValue, PublishRequest} +import conf.ApplicationConfiguration +import model.FeastAppModel.{ContainerItem, FeastAppContainer, FeastAppCuration, Recipe, RecipeIdentifier} +import model.editions.{PublishableIssue, PublishedArticle, PublishedCollection} +import play.api.libs.json.{Json, Writes} +import util.TimestampGenerator +import scala.jdk.CollectionConverters._ + +object FeastPublicationTarget { + object MessageType extends Enumeration { + val Issue, EditionsList = Value + } + type MessageType = MessageType.Value +} + +class FeastPublicationTarget(snsClient:AmazonSNS, config: ApplicationConfiguration, timestamp:TimestampGenerator) extends PublicationTarget { + private def transformArticles(source:PublishedArticle):ContainerItem = { + //FIXME: This is a hack, since we can't actually generate any of the content types that Feast wants yet! + Recipe(recipe = RecipeIdentifier(source.internalPageCode.toString)) + } + + private val findSpace = "\\s+".r + + /** + * Feast app expects a name like `all-recipes` wheras we have `All Recipes` + * @param originalName name to transform + * @return name in kebab-case + */ + private def transformName(originalName:String):String = findSpace.replaceAllIn(originalName.toLowerCase, "-") + + private def transformCollections(collection:PublishedCollection):FeastAppContainer = + FeastAppContainer( + id=collection.id, + title=collection.name, + body=Some(""), //TBD, this is just how it appears in the data at the moment + items = collection.items.map(transformArticles) + ) + + def transformContent(source: PublishableIssue): FeastAppCuration = { + FeastAppCuration( + id=source.id, + issueDate=source.issueDate, + edition=source.edition, + version=source.version, + fronts=source.fronts.map(f=>{ + (transformName(f.name), f.collections.map(transformCollections).toIndexedSeq) + }).toMap + ) + } + + override def putIssue(issue: PublishableIssue, key: Option[String]=None): Unit = { + val outputKey = key.getOrElse(EditionsBucket.createKey(issue)) + putIssueJson(transformContent(issue), outputKey) + } + + private def createPublishRequest(content:String, messageType:FeastPublicationTarget.MessageType):PublishRequest = { + new PublishRequest() + .withMessage(content) + .withTopicArn(config.aws.feastAppPublicationTopic) + .withMessageAttributes(Map( + "timestamp"->new MessageAttributeValue().withDataType("Number").withStringValue(timestamp.getTimestamp.toString), + "type"->new MessageAttributeValue().withDataType("String").withStringValue(messageType.toString), + ).asJava) + } + + override def putIssueJson[T:Writes](issue: T, key:String): Unit = { + val content = Json.stringify(Json.toJson(issue)) + snsClient.publish(createPublishRequest(content, FeastPublicationTarget.MessageType.Issue)) + } + + override def putEditionsList(rawJson: String): Unit = { + snsClient.publish(createPublishRequest(rawJson, FeastPublicationTarget.MessageType.EditionsList)) + } +} diff --git a/app/services/editions/publishing/PublicationTarget.scala b/app/services/editions/publishing/PublicationTarget.scala new file mode 100644 index 00000000000..b5d47953371 --- /dev/null +++ b/app/services/editions/publishing/PublicationTarget.scala @@ -0,0 +1,17 @@ +package services.editions.publishing + +import model.editions.PublishableIssue +import play.api.libs.json.Writes +import PublishedIssueFormatters._ + +trait PublicationTarget { + def putIssue(issue: PublishableIssue, key:Option[String] = None): Unit + + protected def putIssueJson[C: Writes](content: C, key:String): Unit + + //FIXME: It seems strange and awkward to have one method which puts structured data and another that does the same + //but with an arbitrary string, making it unchecked. This can probably be pulled into `putIssueJson` but I think that + // is out of scope for this PR + def putEditionsList(rawJson: String): Unit +} + diff --git a/app/services/editions/publishing/Publishing.scala b/app/services/editions/publishing/Publishing.scala new file mode 100644 index 00000000000..f2a84011001 --- /dev/null +++ b/app/services/editions/publishing/Publishing.scala @@ -0,0 +1,96 @@ +package services.editions.publishing + +import java.time.OffsetDateTime +import com.gu.pandomainauth.model.User +import logging.Logging +import model.editions.Edition.{FeastNorthernHemisphere, FeastSouthernHemisphere} +import model.editions.{EditionsIssue, PublishAction} +import net.logstash.logback.marker.Markers +import services.editions.db.EditionsDB +import play.api.libs.json.Writes + +import scala.jdk.CollectionConverters._ + +class Publishing(editionsAppPublicationBucket: EditionsBucket, + editionsAppPreviewBucket: EditionsBucket, + feastAppPublicationTarget: FeastPublicationTarget, + db: EditionsDB + ) extends Logging { + + def updatePreview(issue: EditionsIssue) = { + val previewIssue = issue.toPreviewIssue + // Archive a copy + issue.edition match { + case FeastNorthernHemisphere | FeastSouthernHemisphere => + //Feast does not currently support previewing, but we don't need to be reminded of that every time someone drag/drops something! + //Preview will be implemented, when it exists. + () + case _ => + editionsAppPreviewBucket.putIssue(previewIssue) + } + } + + def proof(issue: EditionsIssue, user: User, now: OffsetDateTime) = { + val action = PublishAction.proof + + val versionId = db.createIssueVersion(issue.id, user, now) + + val markers = Markers.appendEntries( + Map( + "issue-action" -> action.toString, + "issue-id" -> issue.id, + "issue-date" -> issue.issueDate.toString, + "version" -> versionId, + "user" -> user.email + ).asJava + ) + + logger.info(s"Uploading $action request for issue ${issue.id} to S3")(markers) + + val publishedIssue = issue.toPublishableIssue(versionId, action) + + // Archive a copy + issue.edition match { + case FeastNorthernHemisphere | FeastSouthernHemisphere => + feastAppPublicationTarget.putIssue(publishedIssue) + case _ => + editionsAppPublicationBucket.putIssue(publishedIssue) + } + } + + def putEditionsList(rawJson: String) = { + editionsAppPublicationBucket.putEditionsList(rawJson) + } + + def publish(issue: EditionsIssue, user: User, version: String) = { + + val action = PublishAction.publish + + val markers = Markers.appendEntries( + Map( + "issue-action" -> action.toString, + "issue-id" -> issue.id, + "issue-date" -> issue.issueDate.toString, + "version" -> version, + "user" -> user.email + ).asJava + ) + + logger.info(s"Uploading $action request for issue ${issue.id} to S3")(markers) + + val publishedIssue = if(!issue.supportsProofing) { + val newVersion = db.createIssueVersion(issue.id, user, OffsetDateTime.now()) + issue.toPublishableIssue(newVersion, PublishAction.proof) //Not very self-explanatory; the use of `PublishAction.proof` here means "build the issue afresh". + } else { + issue.toPublishableIssue(version, PublishAction.publish) //PublishAction.publish here means "only use the previously proofed issue" + } + + // Archive a copy + issue.edition match { + case FeastNorthernHemisphere | FeastSouthernHemisphere => + feastAppPublicationTarget.putIssue(publishedIssue) + case _ => + editionsAppPublicationBucket.putIssue(publishedIssue) + } + } +} diff --git a/app/util/TimestampGenerator.scala b/app/util/TimestampGenerator.scala new file mode 100644 index 00000000000..e0c9bed8f7d --- /dev/null +++ b/app/util/TimestampGenerator.scala @@ -0,0 +1,15 @@ +package util + +import java.time.Instant + +trait TimestampGenerator { + def getTimestamp:Long +} + +class TimestampGeneratorImpl extends TimestampGenerator { + def getTimestamp = Instant.now().toEpochMilli +} + +object TimestampGenerator { + def apply() = new TimestampGeneratorImpl() +} diff --git a/build.sbt b/build.sbt index 0091256e8b9..90066ec0776 100644 --- a/build.sbt +++ b/build.sbt @@ -109,6 +109,7 @@ libraryDependencies ++= Seq( "com.beust" % "jcommander" % "1.75", "org.scalatest" %% "scalatest" % "3.0.8" % "test", + "org.mockito" % "mockito-core" % "5.11.0" % Test ) val UsesDatabaseTest = config("database-int") extend Test diff --git a/conf/application.conf b/conf/application.conf index 19d8218d6b4..81d7226647f 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -124,6 +124,10 @@ PROD { faciatool.show_test_containers=true +feast_app.publication_topic = "arn:aws:sns:eu-west-1:163592447864:facia-CODE-FeastPublicationTopic-PUEUpxdjqJ45" +PROD { + feast_app.publication_topic = "arn:aws:sns:eu-west-1:163592447864:facia-PROD-FeastPublicationTopic-PvqrV1NwT7OA" +} include "local.conf" include file("/etc/gu/facia-tool.application.secrets.conf") diff --git a/conf/local.conf b/conf/local.conf index 5b3ae62d6bb..d62187ee7b0 100644 --- a/conf/local.conf +++ b/conf/local.conf @@ -4,4 +4,3 @@ db.default { password = faciatool port = 4724 } - diff --git a/conf/logback.xml b/conf/logback.xml index f6724af0dae..0546b96c480 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -16,6 +16,9 @@ + diff --git a/fronts-client/src/components/EditionFeedSectionHeader.tsx b/fronts-client/src/components/EditionFeedSectionHeader.tsx index 8ea921773c9..bbd1c6652e5 100644 --- a/fronts-client/src/components/EditionFeedSectionHeader.tsx +++ b/fronts-client/src/components/EditionFeedSectionHeader.tsx @@ -95,16 +95,18 @@ class EditionFeedSectionHeader extends React.Component { - + {editionsIssue.supportsProofing && ( + + )}