feat: add new assertVisual command

maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt

@@ -423,6 +423,25 @@ data class AssertWithAICommand(
     }
 }
 
+data class AssertVisualCommand(
+    val baseline: String,
+    val thresholdPercentage: Int,
+    override val optional: Boolean = false,
+    override val label: String? = null,
+) : Command {
+    override fun description(): String {
+        return label ?: "Assert visual difference with baseline $baseline (threshold: $thresholdPercentage%)"
+    }
+
+    override fun evaluateScripts(jsEngine: JsEngine): Command {
+        return copy(
+            baseline = baseline.evaluateScripts(jsEngine)
+        )
+    }
+}
+
+
+
 data class InputTextCommand(
     val text: String,
     override val label: String? = null,

maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt

@@ -36,6 +36,7 @@ data class MaestroCommand(
     val backPressCommand: BackPressCommand? = null,
     @Deprecated("Use assertConditionCommand") val assertCommand: AssertCommand? = null,
     val assertConditionCommand: AssertConditionCommand? = null,
+    val assertVisualCommand: AssertVisualCommand? = null,
     val assertNoDefectsWithAICommand: AssertNoDefectsWithAICommand? = null,
     val assertWithAICommand: AssertWithAICommand? = null,
     val inputTextCommand: InputTextCommand? = null,
@@ -82,6 +83,7 @@ data class MaestroCommand(
         assertWithAICommand = command as? AssertWithAICommand,
         inputTextCommand = command as? InputTextCommand,
         inputRandomTextCommand = command as? InputRandomCommand,
+        assertVisualCommand = command as? AssertVisualCommand,
         launchAppCommand = command as? LaunchAppCommand,
         applyConfigurationCommand = command as? ApplyConfigurationCommand,
         openLinkCommand = command as? OpenLinkCommand,
@@ -137,6 +139,7 @@ data class MaestroCommand(
         clearKeychainCommand != null -> clearKeychainCommand
         runFlowCommand != null -> runFlowCommand
         setLocationCommand != null -> setLocationCommand
+        assertVisualCommand != null -> assertVisualCommand
         repeatCommand != null -> repeatCommand
         copyTextCommand != null -> copyTextCommand
         pasteTextCommand != null -> pasteTextCommand

maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt

@@ -19,6 +19,8 @@
 
 package maestro.orchestra
 
+import com.github.romankh3.image.comparison.ImageComparison
+import com.github.romankh3.image.comparison.model.ImageComparisonState
 import kotlinx.coroutines.runBlocking
 import maestro.*
 import maestro.Filters.asFilter
@@ -46,8 +48,15 @@ import okio.Buffer
 import okio.Sink
 import okio.buffer
 import okio.sink
+import java.awt.image.BufferedImage
 import java.io.File
+import java.io.IOException
 import java.lang.Long.max
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import javax.imageio.ImageIO
 
 // TODO(bartkepacia): Use this in onCommandGeneratedOutput.
 //  Caveat:
@@ -267,6 +276,7 @@ class Orchestra(
             is PasteTextCommand -> pasteText()
             is SwipeCommand -> swipeCommand(command)
             is AssertCommand -> assertCommand(command)
+            is AssertVisualCommand -> assertVisualCommand(command)
             is AssertConditionCommand -> assertConditionCommand(command)
             is AssertNoDefectsWithAICommand -> assertNoDefectsWithAICommand(command)
             is AssertWithAICommand -> assertWithAICommand(command)
@@ -406,6 +416,51 @@ class Orchestra(
         false
     }
 
+    private fun assertVisualCommand(command: AssertVisualCommand): Boolean {
+        val baseline = command.baseline + ".png"
+        val thresholdDifferencePercentage = (100 - command.thresholdPercentage).toDouble()
+
+        val screenshotsDir = File(".maestro/visual_regression").apply { mkdirs() }
+
+        val actual = File(screenshotsDir, baseline)
+
+        val expected = File
+            .createTempFile("screenshot-${System.currentTimeMillis()}", ".png")
+            .also { it.deleteOnExit() }
+
+        maestro.takeScreenshot(expected.sink(), false)
+
+        if (!actual.exists()) {
+            expected.copyTo(actual, overwrite = true)
+            return true
+        }
+
+        val photoNow: BufferedImage = ImageIO.read(expected)
+        val oldPhoto: BufferedImage = ImageIO.read(actual)
+        val failedRegressionDir = File(".maestro/failed_visual_regression").apply { mkdirs() }
+        val regressionFailedFile = File(failedRegressionDir, baseline)
+        val comparison =
+            ImageComparison(photoNow, oldPhoto, regressionFailedFile)
+
+        comparison.apply {
+            allowingPercentOfDifferentPixels = thresholdDifferencePercentage
+            rectangleLineWidth = 10
+            pixelToleranceLevel = 50.00
+            minimalRectangleSize = 40
+        }
+
+        val comparisonState = comparison.compareImages()
+
+        if (ImageComparisonState.MISMATCH === comparisonState.imageComparisonState) {
+            throw MaestroException.AssertionFailure(
+                message = "Comparison error: ${command.description()} - threshold not met, current: ${100 - comparisonState.differencePercent}",
+                hierarchyRoot = maestro.viewHierarchy().root,
+            )
+        }
+        return true
+    }
+
+
     private fun evalScriptCommand(command: EvalScriptCommand): Boolean {
         command.scriptString.evaluateScripts(jsEngine)
 

maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlAssertVisual.kt

@@ -0,0 +1,24 @@
+package maestro.orchestra.yaml
+
+import com.fasterxml.jackson.annotation.JsonCreator
+import java.lang.UnsupportedOperationException
+
+private const val DEFAULT_DIFF_THRESHOLD = 95
+
+data class YamlAssertVisual(
+    val baseline: String,
+    val thresholdPercentage: Int = DEFAULT_DIFF_THRESHOLD,
+    val label: String? = null,
+    val optional: Boolean = false,
+) {
+
+    companion object {
+        @JvmStatic
+        @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+        fun parse(baseline: String): YamlAssertVisual {
+            return YamlAssertVisual(
+                baseline = baseline
+            )
+        }
+    }
+}

maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt

@@ -42,6 +42,7 @@ data class YamlFluentCommand(
     val assertNotVisible: YamlElementSelectorUnion? = null,
     val assertTrue: YamlAssertTrue? = null,
     val assertNoDefectsWithAI: YamlAssertNoDefectsWithAI? = null,
+    val assertVisual: YamlAssertVisual? = null,
     val assertWithAI: YamlAssertWithAI? = null,
     val back: YamlActionBack? = null,
     val clearKeychain: YamlActionClearKeychain? = null,
@@ -137,6 +138,16 @@ data class YamlFluentCommand(
                     )
                 )
             )
+            assertVisual != null -> listOf(
+                MaestroCommand(
+                    AssertVisualCommand(
+                        baseline = assertVisual.baseline,
+                        thresholdPercentage = assertVisual.thresholdPercentage,
+                        optional = assertVisual.optional,
+                        label = assertVisual.label
+                    )
+                )
+            )
             addMedia != null -> listOf(
                 MaestroCommand(
                     addMediaCommand = addMediaCommand(addMedia, flowPath)