diff --git a/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt b/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt index 1cc4b1f244..e582883e0e 100644 --- a/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt +++ b/maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt @@ -423,6 +423,25 @@ data class AssertWithAICommand( } } +data class AssertVisualCommand( + val baseline: String, + val thresholdPercentage: Int, + override val optional: Boolean = false, + override val label: String? = null, +) : Command { + override fun description(): String { + return label ?: "Assert visual difference with baseline $baseline (threshold: $thresholdPercentage%)" + } + + override fun evaluateScripts(jsEngine: JsEngine): Command { + return copy( + baseline = baseline.evaluateScripts(jsEngine) + ) + } +} + + + data class InputTextCommand( val text: String, override val label: String? = null, diff --git a/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt b/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt index bb7090f5b3..1f9992179f 100644 --- a/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt +++ b/maestro-orchestra-models/src/main/java/maestro/orchestra/MaestroCommand.kt @@ -36,6 +36,7 @@ data class MaestroCommand( val backPressCommand: BackPressCommand? = null, @Deprecated("Use assertConditionCommand") val assertCommand: AssertCommand? = null, val assertConditionCommand: AssertConditionCommand? = null, + val assertVisualCommand: AssertVisualCommand? = null, val assertNoDefectsWithAICommand: AssertNoDefectsWithAICommand? = null, val assertWithAICommand: AssertWithAICommand? = null, val inputTextCommand: InputTextCommand? = null, @@ -82,6 +83,7 @@ data class MaestroCommand( assertWithAICommand = command as? AssertWithAICommand, inputTextCommand = command as? InputTextCommand, inputRandomTextCommand = command as? InputRandomCommand, + assertVisualCommand = command as? AssertVisualCommand, launchAppCommand = command as? LaunchAppCommand, applyConfigurationCommand = command as? ApplyConfigurationCommand, openLinkCommand = command as? OpenLinkCommand, @@ -137,6 +139,7 @@ data class MaestroCommand( clearKeychainCommand != null -> clearKeychainCommand runFlowCommand != null -> runFlowCommand setLocationCommand != null -> setLocationCommand + assertVisualCommand != null -> assertVisualCommand repeatCommand != null -> repeatCommand copyTextCommand != null -> copyTextCommand pasteTextCommand != null -> pasteTextCommand diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt index 70d233e20e..0eacb6052b 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt @@ -19,6 +19,8 @@ package maestro.orchestra +import com.github.romankh3.image.comparison.ImageComparison +import com.github.romankh3.image.comparison.model.ImageComparisonState import kotlinx.coroutines.runBlocking import maestro.* import maestro.Filters.asFilter @@ -46,8 +48,15 @@ import okio.Buffer import okio.Sink import okio.buffer import okio.sink +import java.awt.image.BufferedImage import java.io.File +import java.io.IOException import java.lang.Long.max +import java.nio.file.Files +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import javax.imageio.ImageIO // TODO(bartkepacia): Use this in onCommandGeneratedOutput. // Caveat: @@ -267,6 +276,7 @@ class Orchestra( is PasteTextCommand -> pasteText() is SwipeCommand -> swipeCommand(command) is AssertCommand -> assertCommand(command) + is AssertVisualCommand -> assertVisualCommand(command) is AssertConditionCommand -> assertConditionCommand(command) is AssertNoDefectsWithAICommand -> assertNoDefectsWithAICommand(command) is AssertWithAICommand -> assertWithAICommand(command) @@ -406,6 +416,51 @@ class Orchestra( false } + private fun assertVisualCommand(command: AssertVisualCommand): Boolean { + val baseline = command.baseline + ".png" + val thresholdDifferencePercentage = (100 - command.thresholdPercentage).toDouble() + + val screenshotsDir = File(".maestro/visual_regression").apply { mkdirs() } + + val actual = File(screenshotsDir, baseline) + + val expected = File + .createTempFile("screenshot-${System.currentTimeMillis()}", ".png") + .also { it.deleteOnExit() } + + maestro.takeScreenshot(expected.sink(), false) + + if (!actual.exists()) { + expected.copyTo(actual, overwrite = true) + return true + } + + val photoNow: BufferedImage = ImageIO.read(expected) + val oldPhoto: BufferedImage = ImageIO.read(actual) + val failedRegressionDir = File(".maestro/failed_visual_regression").apply { mkdirs() } + val regressionFailedFile = File(failedRegressionDir, baseline) + val comparison = + ImageComparison(photoNow, oldPhoto, regressionFailedFile) + + comparison.apply { + allowingPercentOfDifferentPixels = thresholdDifferencePercentage + rectangleLineWidth = 10 + pixelToleranceLevel = 50.00 + minimalRectangleSize = 40 + } + + val comparisonState = comparison.compareImages() + + if (ImageComparisonState.MISMATCH === comparisonState.imageComparisonState) { + throw MaestroException.AssertionFailure( + message = "Comparison error: ${command.description()} - threshold not met, current: ${100 - comparisonState.differencePercent}", + hierarchyRoot = maestro.viewHierarchy().root, + ) + } + return true + } + + private fun evalScriptCommand(command: EvalScriptCommand): Boolean { command.scriptString.evaluateScripts(jsEngine) diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlAssertVisual.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlAssertVisual.kt new file mode 100644 index 0000000000..22bc680919 --- /dev/null +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlAssertVisual.kt @@ -0,0 +1,24 @@ +package maestro.orchestra.yaml + +import com.fasterxml.jackson.annotation.JsonCreator +import java.lang.UnsupportedOperationException + +private const val DEFAULT_DIFF_THRESHOLD = 95 + +data class YamlAssertVisual( + val baseline: String, + val thresholdPercentage: Int = DEFAULT_DIFF_THRESHOLD, + val label: String? = null, + val optional: Boolean = false, +) { + + companion object { + @JvmStatic + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + fun parse(baseline: String): YamlAssertVisual { + return YamlAssertVisual( + baseline = baseline + ) + } + } +} \ No newline at end of file diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt index 447b22ee6c..dd969ddd47 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt @@ -42,6 +42,7 @@ data class YamlFluentCommand( val assertNotVisible: YamlElementSelectorUnion? = null, val assertTrue: YamlAssertTrue? = null, val assertNoDefectsWithAI: YamlAssertNoDefectsWithAI? = null, + val assertVisual: YamlAssertVisual? = null, val assertWithAI: YamlAssertWithAI? = null, val back: YamlActionBack? = null, val clearKeychain: YamlActionClearKeychain? = null, @@ -137,6 +138,16 @@ data class YamlFluentCommand( ) ) ) + assertVisual != null -> listOf( + MaestroCommand( + AssertVisualCommand( + baseline = assertVisual.baseline, + thresholdPercentage = assertVisual.thresholdPercentage, + optional = assertVisual.optional, + label = assertVisual.label + ) + ) + ) addMedia != null -> listOf( MaestroCommand( addMediaCommand = addMediaCommand(addMedia, flowPath)