Merge branch 'main' into rm/migrate-some-utils-to-ts

.github/workflows/ci.yaml

@@ -0,0 +1,35 @@
+name: CI
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  CI:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v2
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }}
+          aws-region: eu-west-1
+      - uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "corretto"
+      - uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: Build Pluto lambda
+        run: |
+          ./scripts/pluto-ci.sh
+      - name: Build Media Atom Maker
+        run: |
+          ./scripts/app-ci.sh
+      - name: Compile Scala and upload artifacts to RiffRaff
+        run: |
+          ./scripts/scala-ci.sh
+
+

app/controllers/Api.scala

@@ -45,15 +45,19 @@ class Api(
     }
   }
 
-  def getMediaAtoms(search: Option[String], limit: Option[Int]) = APIAuthAction {
-    val atoms = stores.atomListStore.getAtoms(search, limit)
+  def getMediaAtoms(search: Option[String], limit: Option[Int], shouldUseCreatedDateForSort: Boolean) = APIAuthAction {
+    val atoms = stores.atomListStore.getAtoms(search, limit, shouldUseCreatedDateForSort)
     Ok(Json.toJson(atoms))
   }
 
-  def getMediaAtom(id: String) = APIAuthAction {
+  def getMediaAtom(id: String) = APIAuthAction {req =>
     try {
+      val maybeCorsValue = req.headers.get("Origin").filter(_.endsWith("gutools.co.uk"))
       val atom = getPreviewAtom(id)
-      Ok(Json.toJson(MediaAtom.fromThrift(atom)))
+      Ok(Json.toJson(MediaAtom.fromThrift(atom))).withHeaders(
+        "Access-Control-Allow-Origin" -> maybeCorsValue.getOrElse(""),
+        "Access-Control-Allow-Credentials" -> maybeCorsValue.isDefined.toString
+      )
     } catch {
       commandExceptionAsResult
     }

app/controllers/VideoUIApp.scala

@@ -1,6 +1,7 @@
 package controllers
 
 
+import com.gu.editorial.permissions.client.{Permission, PermissionDenied, PermissionsUser}
 import com.gu.media.MediaAtomMakerPermissionsProvider
 import com.gu.media.logging.Logging
 import com.gu.media.youtube.YouTubeAccess
@@ -66,6 +67,7 @@ class VideoUIApp(val authActions: HMACAuthActions, conf: Configuration, awsConfi
         title = "Media Atom Maker",
         jsLocation,
         presenceJsLocation = clientConfig.presence.map(_.jsLocation),
+        pinboardJsLocation = if(permissions.pinboard) awsConfig.pinboardLoaderUrl else None,
         Json.toJson(clientConfig).toString(),
         isHotReloading,
         CSRF.getToken.value

app/data/AtomListStore.scala

@@ -11,18 +11,18 @@ import model.{MediaAtomList, MediaAtomSummary}
 import play.api.libs.json.{JsArray, JsValue}
 
 trait AtomListStore {
-  def getAtoms(search: Option[String], limit: Option[Int]): MediaAtomList
+  def getAtoms(search: Option[String], limit: Option[Int], shouldUseCreatedDateForSort: Boolean): MediaAtomList
 }
 
 class CapiBackedAtomListStore(capi: CapiAccess) extends AtomListStore {
-  override def getAtoms(search: Option[String], limit: Option[Int]): MediaAtomList = {
+  override def getAtoms(search: Option[String], limit: Option[Int], shouldUseCreatedDateForSort: Boolean): MediaAtomList = {
     // CAPI max page size is 200
     val cappedLimit: Option[Int] = limit.map(Math.min(200, _))
 
     val base: Map[String, String] = Map(
       "types" -> "media",
       "order-by" -> "newest"
-    )
+    ) ++ (if(shouldUseCreatedDateForSort) Map("order-date" -> "first-publication") else Map.empty)
 
     val baseWithSearch = search match {
       case Some(q) => base ++ Map(
@@ -76,20 +76,24 @@ class CapiBackedAtomListStore(capi: CapiAccess) extends AtomListStore {
 }
 
 class DynamoBackedAtomListStore(store: PreviewDynamoDataStore) extends AtomListStore {
-  override def getAtoms(search: Option[String], limit: Option[Int]): MediaAtomList = {
+  override def getAtoms(search: Option[String], limit: Option[Int], shouldUseCreatedDateForSort: Boolean): MediaAtomList = {
     // We must filter the entire list of atoms rather than use Dynamo limit to ensure stable iteration order.
     // Without it, the front page will shuffle around when clicking the Load More button.
     store.listAtoms match {
       case Left(err) =>
         AtomDataStoreError(err.msg)
 
       case Right(atoms) =>
-        def created(atom: MediaAtom) = atom.contentChangeDetails.created.map(_.date.getMillis)
+        def sortField(atom: MediaAtom) =
+          if(shouldUseCreatedDateForSort)
+            atom.contentChangeDetails.created
+          else
+            atom.contentChangeDetails.lastModified
 
         val mediaAtoms = atoms
           .map(MediaAtom.fromThrift)
           .toList
-          .sortBy(created)
+          .sortBy(sortField(_).map(_.date.getMillis))
           .reverse // newest atoms first
 
         val filteredAtoms = search match {

app/model/commands/PublishAtomCommand.scala

@@ -134,16 +134,15 @@ case class PublishAtomCommand(
       contentChangeDetails = atom.contentChangeDetails.copy(
         published = changeRecord,
         lastModified = changeRecord,
-        revision = atom.contentChangeDetails.revision + 1,
         scheduledLaunch = None,
         embargo = None
       )
     )
 
     AuditMessage(id, "Publish", getUsername(user)).logMessage()
-    UpdateAtomCommand(id, updatedAtom, stores, user, awsConfig).process()
+    val updatedAtomToPublish = UpdateAtomCommand(id, updatedAtom, stores, user, awsConfig).process()
 
-    val publishedAtom = publishAtomToLive(updatedAtom)
+    val publishedAtom = publishAtomToLive(updatedAtomToPublish)
     updateInactiveAssets(publishedAtom)
     publishedAtom
   }

app/util/AWS.scala

@@ -29,6 +29,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden
     .withCredentials(credentials.instance)
     .build()
 
+  lazy val pinboardLoaderUrl = getString("panda.domain").map(domain => s"https://pinboard.$domain/pinboard.loader.js")
   lazy val composerUrl = getMandatoryString("flexible.url")
   lazy val workflowUrl = getMandatoryString("workflow.url")
   lazy val viewerUrl = getMandatoryString("viewer.url")

app/util/ThumbnailGenerator.scala

@@ -1,7 +1,7 @@
 package util
 
 import java.awt.RenderingHints
-import java.awt.image.BufferedImage
+import java.awt.image.{BufferedImage, ColorConvertOp}
 import java.io._
 import java.net.URL
 import com.google.api.client.http.InputStreamContent
@@ -24,8 +24,11 @@ case class ThumbnailGenerator(logoFile: File) extends Logging {
       .maxBy(_.size.get)
   }
 
-  private def imageAssetToBufferedImage(imageAsset: ImageAsset): BufferedImage =
-    ImageIO.read(new URL(imageAsset.file))
+  private def imageAssetToBufferedImage(imageAsset: ImageAsset): BufferedImage = {
+    val image = ImageIO.read(new URL(imageAsset.file))
+    val rgbImage = new BufferedImage(image.getWidth, image.getHeight, BufferedImage.TYPE_3BYTE_BGR)
+    new ColorConvertOp(null).filter(image, rgbImage)
+  }
 
   private def overlayImages(bgImage: BufferedImage, bgImageMimeType: String, atomId: String): ByteArrayInputStream = {
     val logoWidth: Double = List(bgImage.getWidth() / 3.0, logo.getWidth()).min

app/views/VideoUIApp/app.scala.html

@@ -2,6 +2,7 @@
   title: String,
   jsFileLocation: String,
   presenceJsLocation: Option[String],
+  pinboardJsLocation: Option[String],
   clientConfigJson: String,
   isHotReloading: Boolean,
   csrf: String
@@ -22,5 +23,8 @@ <h1>Loading...</h1>
 
     <script src="@jsFileLocation"></script>
 
+    @pinboardJsLocation.map { pinboardJs =>
+      <script async src="@pinboardJs" ></script>
+    }
 
 }

build.sbt

@@ -253,8 +253,6 @@ lazy val root = (project in file("root"))
       (scheduler / Universal / packageBin).value -> s"${(scheduler / name).value}/${(scheduler / Universal / packageBin).value.getName}",
       (app / baseDirectory).value / "pluto-message-ingestion/target/pluto-message-ingestion.zip" -> "pluto-message-ingestion/pluto-message-ingestion.zip",
       (app / baseDirectory).value / "conf/riff-raff.yaml" -> "riff-raff.yaml",
-      (app / baseDirectory).value / "fluentbit/td-agent-bit.conf" -> "media-atom-maker/fluentbit/td-agent-bit.conf",
-      (app / baseDirectory).value / "fluentbit/parsers.conf" -> "media-atom-maker/fluentbit/parsers.conf",
       (uploader / Compile / resourceManaged).value / "media-atom-pipeline.yaml" -> "media-atom-pipeline-cloudformation/media-atom-pipeline.yaml"
     )
   )

cloudformation/media-atom-maker-dev.yml

@@ -96,7 +96,7 @@ Resources:
     Type: "AWS::IAM::AccessKey"
     Properties:
       UserName: {"Ref": "MediaAtomUser"}
-      Serial: 3
+      Serial: 4
   MediaAtomsDynamoTable:
     Type: "AWS::DynamoDB::Table"
     Properties:

common/src/main/scala/com/gu/media/Permissions.scala

@@ -7,25 +7,28 @@ import play.api.libs.json.Format
 import com.gu.pandomainauth.model.{User => PandaUser}
 import scala.concurrent.Future
 
-case class Permissions(deleteAtom: Boolean, addSelfHostedAsset: Boolean, setVideosOnAllChannelsPublic: Boolean)
+case class Permissions(
+  deleteAtom: Boolean,
+  addSelfHostedAsset: Boolean,
+  setVideosOnAllChannelsPublic: Boolean,
+  pinboard: Boolean
+)
 object Permissions {
   implicit val format: Format[Permissions] = Jsonx.formatCaseClass[Permissions]
 
   val app = "atom-maker"
   val deleteAtom = Permission("delete_atom", app, defaultVal = PermissionDenied)
   val addSelfHostedAsset = Permission("add_self_hosted_asset", app, defaultVal = PermissionDenied)
   val setVideosOnAllChannelsPublic = Permission("set_videos_on_all_channels_public", app, defaultVal = PermissionDenied)
+  val pinboard = Permission("pinboard", "pinboard", defaultVal = PermissionDenied)
 }
 
 class MediaAtomMakerPermissionsProvider(stage: String, credsProvider: AWSCredentialsProvider) extends PermissionsProvider {
   import Permissions._
 
-  val all = Seq(deleteAtom, addSelfHostedAsset, setVideosOnAllChannelsPublic)
-  val none = Permissions(deleteAtom = false, addSelfHostedAsset = false, setVideosOnAllChannelsPublic = false )
-
   implicit def config = PermissionsConfig(
     app = "media-atom-maker",
-    all = Seq(deleteAtom, addSelfHostedAsset, setVideosOnAllChannelsPublic),
+    all = Seq(deleteAtom, addSelfHostedAsset, setVideosOnAllChannelsPublic, pinboard),
     s3BucketPrefix = if(stage == "PROD") "PROD" else "CODE",
     awsCredentials = credsProvider
   )
@@ -34,19 +37,13 @@ class MediaAtomMakerPermissionsProvider(stage: String, credsProvider: AWSCredent
     deleteAtom <- hasPermission(deleteAtom, user)
     selfHostedMediaAtom <- hasPermission(addSelfHostedAsset, user)
     publicStatusPermissions <- hasPermission(setVideosOnAllChannelsPublic, user)
-  } yield Permissions(deleteAtom, selfHostedMediaAtom, publicStatusPermissions)
-
-
-  def getUploadPermissions(user: PandaUser): Future[Permissions] = for {
-    selfHostedMediaAtom <- hasPermission(addSelfHostedAsset, user)
-  } yield {
-    Permissions(deleteAtom = false, selfHostedMediaAtom, setVideosOnAllChannelsPublic = false)
-  }
+    pinboard <- hasPermission(pinboard, user)
+  } yield Permissions(deleteAtom, selfHostedMediaAtom, publicStatusPermissions, pinboard)
 
   def getStatusPermissions(user: PandaUser): Future[Permissions] = for {
     publicStatus <- hasPermission(setVideosOnAllChannelsPublic, user)
   } yield {
-    Permissions(deleteAtom = false, addSelfHostedAsset = false, publicStatus)
+    Permissions(deleteAtom = false, addSelfHostedAsset = false, publicStatus, pinboard = false)
   }
 
   private def hasPermission(permission: Permission, user: PandaUser): Future[Boolean] = {

conf/riff-raff.yaml

@@ -12,7 +12,7 @@ deployments:
       app: media-atom-maker
       parameters:
         amiTags:
-          Recipe: editorial-tools-focal-java8-ARM
+          Recipe: editorial-tools-focal-java8-ARM-WITH-cdk-base
           AmigoStage: PROD
   media-atom-maker:
     type: autoscaling

conf/routes

@@ -1,5 +1,5 @@
 # optional limit
-GET     /api/atoms                     controllers.Api.getMediaAtoms(search: Option[String], limit: Option[Int])
+GET     /api/atoms                     controllers.Api.getMediaAtoms(search: Option[String], limit: Option[Int], shouldUseCreatedDateForSort: Boolean?=false)
 POST    /api/atoms                     controllers.Api.createMediaAtom
 
 GET     /api/atoms/:id                 controllers.Api.getMediaAtom(id)

docs/01-dev-setup.md

@@ -5,20 +5,20 @@ Ensure you have the following installed:
 - awscli
 - Java 8
 - nginx
-- node v10+
+- node v14.18.1
 - npm
 - yarn
 - nvm
 - [dev-nginx](https://github.com/guardian/dev-nginx#installation)
 
 You'll also need Janus credentials to the `media-service` account.
 
-## Local setup 
+## Local setup
 
 We use a shared DEV stack, with a shared config. Fetch it by running:
 
 ```bash
-./scripts/fetch-dev-config.sh
+sudo ./scripts/fetch-dev-config.sh
 ```
 
 There is a chance that the IAM key used for local development (media-atom-maker-DEV) has been disabled if it has not been rotated in a while. If this is the case, and you need the key, you will need to rotate the IAM key. To do this, increment the [Serial property](https://github.com/guardian/media-atom-maker/blob/ba9f87b4b3d3f3446affabc4410ea598ae130e36/cloudformation/media-atom-maker-dev.yml#L99) in the CloudFormation template, and update the stack with the new template. This will generate the new IAM key (found in the CloudFormation `Outputs` tab, under `AwsId` and `AwsSecret`), which you should update in the dev config file in S3 (under the settings `upload.accessKey` and `upload.secretKey`).

public/video-ui/src/actions/SearchActions/updateShouldUseCreatedDateForSort.js

@@ -0,0 +1,7 @@
+export function updateShouldUseCreatedDateForSort(shouldUseCreatedDateForSort) {
+  return {
+    type: 'UPDATE_SHOULD_USE_CREATED_DATE_FOR_SORT',
+    shouldUseCreatedDateForSort,
+    receivedAt: Date.now()
+  };
+}