WIP native visual regression testing

e2e/basic-smoke.spec.ts

@@ -18,11 +18,13 @@ test('should show the home screen', async () => {
 
 test('should show the settings screen after tap', async () => {
  await element(by.id('button-open-settings')).tap()
+ await element(by.text('Settings')).tap()
  await expect(element(by.text('Sign In to St. Olaf'))).toBeVisible()
 })
 
 test('should show home screen after tap to exit settings screen', async () => {
  await element(by.id('button-open-settings')).tap()
+ await element(by.text('Settings')).tap()
  await expect(element(by.id('screen-homescreen'))).not.toBeVisible()
  await element(by.text('Done')).tap()
  await expect(element(by.id('screen-homescreen'))).toBeVisible()

ios/AllAboutOlaf.xcodeproj/project.pbxproj

@@ -17,6 +17,9 @@
  3ACC8B85273335930069E931 /* windmill@2x~iPad.png in Resources */ = {isa = PBXBuildFile; fileRef = 3ACC8B83273335930069E931 /* windmill@2x~iPad.png */; };
  3ACC8B86273335930069E931 /* windmill@3x~iPad.png in Resources */ = {isa = PBXBuildFile; fileRef = 3ACC8B84273335930069E931 /* windmill@3x~iPad.png */; };
  3AE408501E1E280800F0FD83 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3AE4084F1E1E280800F0FD83 /* LaunchScreen.storyboard */; };
+ 3AEA098429904E9F009007FF /* ImageComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AEA098329904E9F009007FF /* ImageComparison.swift */; };
+ 3AEA098729906E66009007FF /* UIImage+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AEA098629906E66009007FF /* UIImage+Extensions.swift */; };
+ 3AEA098929906EE2009007FF /* Structures.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AEA098829906EE2009007FF /* Structures.swift */; };
  8222E4786CCF790474A60080 /* libPods-AllAboutOlaf-AllAboutOlafUITests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2256074D704451D201AB966D /* libPods-AllAboutOlaf-AllAboutOlafUITests.a */; };
  91E0D7C5F14D215A7C718236 /* libPods-AllAboutOlaf.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 9EB28C76433AECBFEDDD439B /* libPods-AllAboutOlaf.a */; };
 /* End PBXBuildFile section */
@@ -334,6 +337,9 @@
  3ACC8B83273335930069E931 /* windmill@2x~iPad.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; name = "windmill@2x~iPad.png"; path = "../images/icons/windmill@2x~iPad.png"; sourceTree = "<group>"; };
  3ACC8B84273335930069E931 /* windmill@3x~iPad.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; name = "windmill@3x~iPad.png"; path = "../images/icons/windmill@3x~iPad.png"; sourceTree = "<group>"; };
  3AE4084F1E1E280800F0FD83 /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = AllAboutOlaf/LaunchScreen.storyboard; sourceTree = "<group>"; };
+ 3AEA098329904E9F009007FF /* ImageComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageComparison.swift; sourceTree = "<group>"; };
+ 3AEA098629906E66009007FF /* UIImage+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIImage+Extensions.swift"; sourceTree = "<group>"; };
+ 3AEA098829906EE2009007FF /* Structures.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Structures.swift; sourceTree = "<group>"; };
  5CBB688C2084ADBF20C737EF /* Pods-AllAboutOlaf-AllAboutOlafUITests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AllAboutOlaf-AllAboutOlafUITests.debug.xcconfig"; path = "Target Support Files/Pods-AllAboutOlaf-AllAboutOlafUITests/Pods-AllAboutOlaf-AllAboutOlafUITests.debug.xcconfig"; sourceTree = "<group>"; };
  67354E87236E0E5900B1E8E8 /* AllAboutOlaf.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = AllAboutOlaf.app; sourceTree = BUILT_PRODUCTS_DIR; };
  67354E88236E0E5900B1E8E8 /* AllAboutOlafUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = AllAboutOlafUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -438,6 +444,7 @@
  00D33BE01DDA58A8001E830E /* UI Tests */ = {
  isa = PBXGroup;
  children = (
+ 3AEA098529906E31009007FF /* ImageComparison */,
  00D33BE31DDA58A8001E830E /* Info.plist */,
  00D33C011DDA5958001E830E /* SnapshotHelper.swift */,
  00D33BE11DDA58A8001E830E /* AllAboutOlafUITests.swift */,
@@ -512,6 +519,16 @@
  name = Products;
  sourceTree = "<group>";
  };
+ 3AEA098529906E31009007FF /* ImageComparison */ = {
+ isa = PBXGroup;
+ children = (
+ 3AEA098329904E9F009007FF /* ImageComparison.swift */,
+ 3AEA098629906E66009007FF /* UIImage+Extensions.swift */,
+ 3AEA098829906EE2009007FF /* Structures.swift */,
+ );
+ path = ImageComparison;
+ sourceTree = "<group>";
+ };
  3C5BFA21338636E7EB4E5E0D /* Frameworks */ = {
  isa = PBXGroup;
  children = (
@@ -1236,8 +1253,11 @@
  isa = PBXSourcesBuildPhase;
  buildActionMask = 2147483647;
  files = (
+ 3AEA098929906EE2009007FF /* Structures.swift in Sources */,
  00D33BE21DDA58A8001E830E /* AllAboutOlafUITests.swift in Sources */,
  00D33C031DDA598B001E830E /* SnapshotHelper.swift in Sources */,
+ 3AEA098429904E9F009007FF /* ImageComparison.swift in Sources */,
+ 3AEA098729906E66009007FF /* UIImage+Extensions.swift in Sources */,
  );
  runOnlyForDeploymentPostprocessing = 0;
  };

ios/AllAboutOlafUITests/AllAboutOlafUITests.swift

@@ -117,4 +117,35 @@ class AllAboutOlafUITests: XCTestCase {
  sleep(5)
  snapshot("13StudentOrgsScreen")
  }
+
+ func testSettingCustomHomeBackgroundImageScreen() {
+ sleep(2)
+
+ // screenshot the initial state
+ XCTAssertTrue(app.buttons["button-open-settings"].isHittable)
+ let before = XCUIScreen.main.screenshot().image
+
+ // open settings and tap the change background button
+ app.buttons["button-open-settings"].tap()
+ app.buttons["Change Background"].tap()
+
+ // ensure we are on the photos tab
+ XCTAssertTrue(app.buttons["Photos"].isHittable)
+ app.buttons["Photos"].tap()
+
+ // get the scrollable photos
+ let photoPredicate = NSPredicate(format: "label BEGINSWITH 'Photo'")
+ let imageElements = app.scrollViews.otherElements.images.containing(photoPredicate)
+
+ // select the second photo in the image library (the first strangely does not load)
+ let secondImage = 1
+ imageElements.element(boundBy: secondImage).tap()
+
+ // screenshot the changed state
+ sleep(2)
+ let after = XCUIScreen.main.screenshot().image
+
+ // visual difference check of before and after with threshold
+ assert(image: after, matches: before, requiredAccuracy: 1, comparisonName: "Custom background image")
+ }
 }

ios/AllAboutOlafUITests/ImageComparison/ImageComparison.swift

@@ -0,0 +1,66 @@
+//
+// UITests+Extensions.swift
+// AllAboutOlafUITests
+//
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-1/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-2/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-3/
+//
+
+import XCTest
+
+func assert(
+ image imageProducer: @autoclosure () -> UIImage,
+ matches reference: UIImage,
+ requiredAccuracy: Double,
+ comparisonName: String,
+ file: StaticString = #file,
+ line: UInt = #line
+) {
+ let image = imageProducer()
+
+ do {
+ try image.ensureMatches(reference: reference, requiredAccuracy: requiredAccuracy)
+ } catch let ImageComparisonError.imageMismatch(
+ pixelCount: totalPixels,
+ acceptableMismatchCount: acceptable,
+ actualMismatchCount: actual
+ ) {
+ XCTContext.runActivity(named: comparisonName) { activity in
+ let imageAttachment = XCTAttachment(image: image)
+ imageAttachment.name = "Actual render"
+ activity.add(imageAttachment)
+
+ let referenceAttachment = XCTAttachment(image: reference)
+ referenceAttachment.name = "Reference image"
+ activity.add(referenceAttachment)
+
+ // New: Generate the difference image and attach it.
+ let fullFrame = CGRect(origin: .zero, size: reference.size)
+ let renderer = UIGraphicsImageRenderer(size: fullFrame.size)
+ let difference = renderer.image { context in
+ // Produce black pixels wherever things are the same.
+ reference.draw(in: fullFrame)
+ image.draw(in: fullFrame, blendMode: .difference, alpha: 1)
+
+ // Highlight everything not black — aka the differences — in red.
+ // This step is optional, but I’ve found it to be helpful in some cases,
+ // e.g. where the differences are subtle and tinted themselves.
+ UIColor.red.withAlphaComponent(0.5).setFill()
+ context.fill(fullFrame, blendMode: .colorDodge)
+ }
+
+ let differenceAttachment = XCTAttachment(image: difference)
+ differenceAttachment.name = "Diff image"
+ activity.add(differenceAttachment)
+
+ XCTFail(
+ "Image with \(totalPixels) pixels differed from reference by \(actual) pixels (allowed: \(acceptable)",
+ file: file,
+ line: line
+ )
+ }
+ } catch{
+ print("Something very bad happened during the visual regression checking and we're not sure how to recover.")
+ }
+}

ios/AllAboutOlafUITests/ImageComparison/Structures.swift

@@ -0,0 +1,47 @@
+//
+// Structures.swift
+// AllAboutOlafUITests
+//
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-1/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-2/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-3/
+//
+
+import Foundation
+
+struct RGBA {
+ let r: UInt8
+ let g: UInt8
+ let b: UInt8
+ let a: UInt8
+
+ func isSimilar(to other: RGBA) -> Bool {
+ (r.distance(to: other.r) < 2
+ && g.distance(to: other.g) < 2
+ && b.distance(to: other.b) < 2
+ && a.distance(to: other.a) < 2)
+ }
+}
+
+/// Comprehensive list of errors that can occur when evaluating a render test.
+enum ImageComparisonError: Error {
+ // :MARK: Test Setup Errors — see last week's article
+ case unsupportedBackingStore(referenceImage: UIImage, actualImage: UIImage)
+ case dimensionMismatch(expectedWidth: Int, expectedHeight: Int, actualWidth: Int, actualHeight: Int)
+
+ // :MARK: Actual Comparison Failure
+ /// The image has the same dimensions as the reference, but too many pixels differ.
+ /// - Parameters:
+ /// - pixelCount:
+ /// The total number of pixels in the image.
+ /// - acceptableMismatchCount:
+ /// The maximum number of pixels that would have been allowed to differ from
+ /// the reference to still be considered a match.
+ /// - actualMismatchCount:
+ /// The actual number of pixels in the image that differed from the reference.
+ case imageMismatch(
+ pixelCount: Int,
+ acceptableMismatchCount: Int,
+ actualMismatchCount: Int
+ )
+}

ios/AllAboutOlafUITests/ImageComparison/UIImage+Extensions.swift

@@ -0,0 +1,131 @@
+//
+// UIImage+Extensions.swift
+// AllAboutOlafUITests
+//
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-1/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-2/
+// https://pspdfkit.com/blog/2021/swift-render-tests-in-practice-part-3/
+//
+
+import XCTest
+import UIKit
+
+extension UIImage {
+
+ func ensureMatches(reference: UIImage, requiredAccuracy: Double) throws {
+ // For simplicity, require images to be backed by Core Graphics.
+ // This will be true for most images. Notable exceptions are images
+ // obtained from Core Image.
+ guard
+ let pixels = cgImage,
+ let referencePixels = reference.cgImage
+ else {
+ throw ImageComparisonError.unsupportedBackingStore(
+ referenceImage: reference,
+ actualImage: self
+ )
+ }
+
+ // We cannot compare images that don't have the same dimensions.
+ guard
+ referencePixels.width == pixels.width,
+ referencePixels.height == pixels.height
+ else {
+ throw ImageComparisonError.dimensionMismatch(
+ expectedWidth: referencePixels.width,
+ expectedHeight: referencePixels.height,
+ actualWidth: pixels.width,
+ actualHeight: pixels.height
+ )
+ }
+
+ // Unfortunately, Core Graphics doesn’t offer direct access to the
+ // underlying uncompressed buffer for an image. We can use this to
+ // our advantage to give us some flexibility when it comes to
+ // storing/representing images for different purposes, which we’ll do
+ // in a helper function.
+ let deviceRGB = CGColorSpaceCreateDeviceRGB()
+ let imageBuffer = try makeRGBABuffer(
+ for: pixels,
+ colorSpace: deviceRGB
+ )
+ // Since the function returns an unmanaged buffer — which makes sense
+ // for what we want — never let it escape without deallocating it!
+ defer {
+ imageBuffer.deallocate()
+ }
+ let referenceBuffer = try makeRGBABuffer(
+ for: referencePixels,
+ colorSpace: deviceRGB
+ )
+ defer {
+ referenceBuffer.deallocate()
+ }
+
+ // We already checked this above, but in case we made a mistake in our
+ // helper, fail here.
+ let pixelCount = imageBuffer.count
+ precondition(
+ pixelCount == referenceBuffer.count,
+ "Cannot compare contents of buffer that differ in size"
+ )
+ // Now we iterate over all the pixels, counting the mismatches.
+ var mismatchingPixelCount = 0
+ for i in 0..<pixelCount {
+ if !imageBuffer[i].isSimilar(to: referenceBuffer[i]) {
+ mismatchingPixelCount += 1
+ }
+ }
+
+ // Finally, throw an error if the number of mismatches exceeds what the caller allowed.
+ let acceptableMismatchCount = Int(
+ Double(pixelCount) * (1 - requiredAccuracy)
+ )
+ if acceptableMismatchCount < mismatchingPixelCount {
+ throw ImageComparisonError.imageMismatch(
+ pixelCount: pixelCount,
+ acceptableMismatchCount: acceptableMismatchCount,
+ actualMismatchCount: mismatchingPixelCount
+ )
+ }
+ }
+
+ private func makeRGBABuffer(for image: CGImage, colorSpace: CGColorSpace) -> UnsafeMutableBufferPointer<RGBA>
+ {
+ precondition(
+ colorSpace.numberOfComponents == 3
+ && colorSpace.model == .rgb,
+ "For RGBA, we need a compatible colorspace"
+ )
+
+ let bitsPerComponent = 8
+ let bytesPerPixel = 4 // 3 components plus alpha.
+ let width = image.width
+ let height = image.height
+
+ // We make a buffer that can fit all the pixels to back a `CGContext`.
+ let imageBuffer = UnsafeMutableBufferPointer<RGBA>
+ .allocate(capacity: width * height)
+
+ // And we make sure the buffer is empty.
+ imageBuffer.initialize(repeating: .init(r: 0, g: 0, b: 0, a: 0))
+
+ // Since we’ve handpicked the configuration, we know this can’t fail.
+ let context = CGContext(
+ data: imageBuffer.baseAddress,
+ width: width,
+ height: height,
+ bitsPerComponent: bitsPerComponent,
+ bytesPerRow: width * bytesPerPixel,
+ space: colorSpace,
+ bitmapInfo: (
+ CGImageAlphaInfo.premultipliedFirst.rawValue
+ | CGBitmapInfo.byteOrder32Big.rawValue)
+ )!
+
+ // Draw the image into the context to get the pixel data into our buffer.
+ context.draw(image, in: .init(x: 0, y: 0, width: width, height: height))
+
+ return imageBuffer
+ }
+}