Fix the acceptance tests

[closes #5]

    - All build and test actions are now done through make
    - To run the acceptance tests, run `make acceptance-test`
            - This has the side effect of creating a file that contains the results of running Muter on the ExampleApp codebase (`muters_output.txt`)
            - The unit tests were refactored to load `muters_output.txt` from disk, rather than have them launch a Muter process
            - Subsequent unit test runs will go green if `muters_output.txt` exists and there are no issues with Muter's code, and can be run using `make test`

Makefile

@@ -5,10 +5,16 @@ libdir = $(prefix)/lib
 REPODIR = $(shell pwd)
 BUILDDIR = $(REPODIR)/.build
 
-build:
+build: 
+ swift build
+
+build-release:
  swift build -c release --disable-sandbox
 
-install: build
+build-tests: 
+ swift build --target muterTests 
+
+install: build-release
  install -d "$(bindir)" "$(libdir)"
  install "$(BUILDDIR)/release/muter" "$(bindir)"
  install "$(BUILDDIR)/release/libSwiftSyntax.dylib" "$(libdir)"
@@ -24,4 +30,10 @@ uninstall:
 clean:
  rm -rf .build
 
-.PHONY: build install uninstall clean
+test: 
+ @swift test # Also builds app and test code
+
+acceptance-test: build
+ ./Scripts/runAcceptanceTests.sh
+
+.PHONY: build install uninstall clean test build-tests

Scripts/runAcceptanceTests.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+echo "📴📴📴📴📴📴📴 Acceptance Testing has started 📴📴📴📴📴📴📴"
+
+echo "Setting up environment for testing..."
+echo "" > ./Tests/muterTests/muters_output.txt # Clear out the results of the last run of Muter
+git checkout -- ExampleApp/ExampleApp/* # Ensures there are no leftover or accidental changes from a prior run of Muter
+cd ./ExampleApp
+rm -rf muter_tmp # Remove Muter's working directory
+
+echo "Running Muter..."
+../.build/x86_64-apple-macosx10.10/debug/muter >> ../Tests/muterTests/muters_output.txt 
+
+echo "Running tests..."
+swift test
+
+echo "📳📳📳📳📳📳📳 Acceptance Testing has finished 📳📳📳📳📳📳📳"

Tests/muterTests/AcceptanceTests.swift

@@ -0,0 +1,73 @@
+@testable import muterCore
+
+import SwiftSyntax
+import XCTest
+
+@available(OSX 10.13, *)
+class AcceptanceTests: XCTestCase {
+ static var originalSourceCode: SourceFileSyntax!
+ static var sourceCodePath: String!
+ static var output: String!
+
+ override static func setUp() {
+ sourceCodePath = "\(exampleAppDirectory)/ExampleApp/Module.swift"
+ originalSourceCode = sourceCode(fromFileAt: sourceCodePath)!
+
+ output = muterOutput
+ }
+
+ func test_muterReportsTheFilesItDiscovers() {
+ XCTAssert(AcceptanceTests.output.contains("Discovered 3 Swift files"), "Muter reports the number of Swift files it discovers, taking into account a blacklist which causes it to ignore certain files or directories")
+ XCTAssertGreaterThanOrEqual(numberOfDiscoveredFileLists(in: AcceptanceTests.output), 1, "Muter lists the paths of Swift files it discovers")
+ }
+
+ func test_muterReportsTheMutationsItCanApply() {
+ XCTAssert(AcceptanceTests.output.contains("Discovered 8 mutations to introduce"), "Muter reports how many mutations it's able to perform")
+ }
+
+ func test_muterPerformsAMutationTest() throws {
+ XCTAssert(AcceptanceTests.output.contains("Mutation Test Passed"), "Muter causes a test suite to fail, which causes the mutation test to pass")
+ XCTAssert(AcceptanceTests.output.contains("Mutation Test Failed"), "Not every mutation test will pass - it depends on the rigor of the test suite under test.")
+ }
+
+ func test_muterReportsAMutationScore() {
+ XCTAssert(AcceptanceTests.output.contains("Mutation Score of Test Suite (higher is better): 25/100"), "Muter reports a mutation score so an engineer can determine how effective their test suite is at identifying defects or changes to a code base")
+ }
+
+ func test_muterCleansUpAfterItself() {
+ let afterSourceCode = sourceCode(fromFileAt: AcceptanceTests.sourceCodePath)
+ let workingDirectoryExists = FileManager.default.fileExists(atPath: "\(AcceptanceTests.exampleAppDirectory)/muter_tmp", isDirectory: nil)
+
+ XCTAssertNotNil(afterSourceCode, "This file should be available - Muter may have accidentally moved or deleted it")
+ XCTAssertEqual(AcceptanceTests.originalSourceCode!.description, afterSourceCode!.description, "Muter is supposed to clean up after itself by restoring the source code it mutates once it's done")
+ XCTAssertFalse(workingDirectoryExists, "Muter is supposed to clean up after itself by deleting the working directory it creates")
+ }
+}
+
+@available(OSX 10.13, *)
+private extension AcceptanceTests {
+ static var exampleAppDirectory: String {
+ return AcceptanceTests().productsDirectory
+ .deletingLastPathComponent()
+ .deletingLastPathComponent()
+ .deletingLastPathComponent() // Go up 3 directories
+ .appendingPathComponent("ExampleApp") // Go down 1 directory
+ .withoutScheme() // Remove the file reference scheme
+ .absoluteString
+ }
+
+ static var muterOutputPath: String { return "\(AcceptanceTests().testDirectory)/muters_output.txt" }
+
+ static var muterOutput: String {
+ let data = FileManager.default.contents(atPath: muterOutputPath)!
+ return String(data: data, encoding: .utf8) ?? ""
+ }
+
+ func numberOfDiscoveredFileLists(in output: String) -> Int {
+ let filePathRegex = try! NSRegularExpression(pattern: "Discovered \\d* Swift files:\n\n(/[^/ ]*)+/?", options: .anchorsMatchLines)
+ let entireString = NSRange(location: 0, length: output.count)
+ return filePathRegex.numberOfMatches(in: output,
+ options: .withoutAnchoringBounds,
+ range: entireString)
+ }
+}