diff --git a/xctool/xctool-tests/RunTestsActionTests.m b/xctool/xctool-tests/RunTestsActionTests.m index 5bec01d91..0649db584 100644 --- a/xctool/xctool-tests/RunTestsActionTests.m +++ b/xctool/xctool-tests/RunTestsActionTests.m @@ -32,6 +32,29 @@ #import "XCToolUtil.h" #import "XcodeSubjectInfo.h" +static BOOL areEqualJsonOutputsIgnoringKeys(NSString *output1, NSString *output2, NSArray *keys) +{ + NSArray *output1Array = [[output1 stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] componentsSeparatedByString:@"\n"]; + NSArray *output2Array = [[output2 stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] componentsSeparatedByString:@"\n"]; + if ([output1Array count] != [output2Array count]) { + return NO; + } + + for (int i=0; i<[output1Array count]; i++) { + NSMutableDictionary *dict1 = [[NSJSONSerialization JSONObjectWithData:[output1Array[i] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] mutableCopy]; + NSMutableDictionary *dict2 = [[NSJSONSerialization JSONObjectWithData:[output2Array[i] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] mutableCopy]; + for (NSString *key in keys) { + [dict1 removeObjectForKey:key]; + [dict2 removeObjectForKey:key]; + } + if (![dict1 isEqual:dict2]) { + return NO; + } + } + + return YES; +} + @interface RunTestsActionTests : SenTestCase @end @@ -274,6 +297,48 @@ - (void)testRunTestsAction }]; } +- (void)testRunTestsActionWithListTestsOnlyOption +{ + NSArray *testList = @[@"TestProject_LibraryTests/testOutputMerging", + @"TestProject_LibraryTests/testPrintSDK", + @"TestProject_LibraryTests/testStream", + @"TestProject_LibraryTests/testWillFail", + @"TestProject_LibraryTests/testWillPass"]; + + [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{ + [[FakeTaskManager sharedManager] addLaunchHandlerBlocks:@[ + // Make sure -showBuildSettings returns some data + [LaunchHandlers handlerForShowBuildSettingsWithProject:TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj" + scheme:@"TestProject-Library" + settingsPath:TEST_DATA @"TestProject-Library-showBuildSettings.txt"], + // We're going to call -showBuildSettings on the test target. + [LaunchHandlers handlerForShowBuildSettingsWithProject:TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj" + target:@"TestProject-LibraryTests" + settingsPath:TEST_DATA @"TestProject-Library-TestProject-LibraryTests-showBuildSettings.txt" + hide:NO], + [LaunchHandlers handlerForOtestQueryReturningTestList:testList], + ]]; + + XCTool *tool = [[[XCTool alloc] init] autorelease]; + + tool.arguments = @[@"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj", + @"-scheme", @"TestProject-Library", + @"-configuration", @"Debug", + @"-sdk", @"iphonesimulator6.0", + @"run-tests", + @"listTestsOnly", + @"-reporter", @"json-stream" + ]; + + NSDictionary *result = [TestUtil runWithFakeStreams:tool]; + NSString *listTestsOnlyOutput = [NSString stringWithContentsOfFile:TEST_DATA @"TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt" + encoding:NSUTF8StringEncoding + error:nil]; + NSString *stdoutString = result[@"stdout"]; + assertThatBool(areEqualJsonOutputsIgnoringKeys(stdoutString, listTestsOnlyOutput, @[@"timestamp", @"duration"]), equalToBool(YES)); + }]; +} + - (void)testCanRunTestsAgainstDifferentTestSDK { NSArray *testList = @[@"TestProject_LibraryTests/testBacktraceOutputIsCaptured", diff --git a/xctool/xctool-tests/TestData/TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt b/xctool/xctool-tests/TestData/TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt new file mode 100644 index 000000000..eb2d92485 --- /dev/null +++ b/xctool/xctool-tests/TestData/TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt @@ -0,0 +1,18 @@ +{"message":"Loading settings for scheme 'TestProject-Library' ...","timestamp":1398211102.696059,"level":"Info","event":"begin-status"} +{"message":"Loading settings for scheme 'TestProject-Library' ...","timestamp":1398211102.719442,"level":"Info","event":"end-status"} +{"scheme":"TestProject-Library","workspace":null,"timestamp":1398211102.719507,"event":"begin-action","project":"xctool-tests\/TestData\/TestProject-Library\/TestProject-Library.xcodeproj","name":"run-tests"} +{"message":"Collecting info for testables...","timestamp":1398211102.719552,"level":"Info","event":"begin-status"} +{"message":"Collecting info for testables...","timestamp":1398211102.803515,"level":"Info","event":"end-status"} +{"bundleName":"TestProject-LibraryTests.octest","sdkName":"iphonesimulator6.0","timestamp":1398211102.803576,"event":"begin-ocunit","targetName":"TestProject-LibraryTests","testType":"logic-test"} +{"test":"-[TestProject_LibraryTests testOutputMerging]","className":"TestProject_LibraryTests","timestamp":1398211102.803618,"event":"begin-test","methodName":"testOutputMerging"} +{"result":"success","test":"-[TestProject_LibraryTests testOutputMerging]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testOutputMerging","totalDuration":"0","timestamp":1398211102.803637} +{"test":"-[TestProject_LibraryTests testPrintSDK]","className":"TestProject_LibraryTests","timestamp":1398211102.803671,"event":"begin-test","methodName":"testPrintSDK"} +{"result":"success","test":"-[TestProject_LibraryTests testPrintSDK]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testPrintSDK","totalDuration":"0","timestamp":1398211102.803688} +{"test":"-[TestProject_LibraryTests testStream]","className":"TestProject_LibraryTests","timestamp":1398211102.803714,"event":"begin-test","methodName":"testStream"} +{"result":"success","test":"-[TestProject_LibraryTests testStream]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testStream","totalDuration":"0","timestamp":1398211102.803728} +{"test":"-[TestProject_LibraryTests testWillFail]","className":"TestProject_LibraryTests","timestamp":1398211102.803743,"event":"begin-test","methodName":"testWillFail"} +{"result":"success","test":"-[TestProject_LibraryTests testWillFail]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testWillFail","totalDuration":"0","timestamp":1398211102.803755} +{"test":"-[TestProject_LibraryTests testWillPass]","className":"TestProject_LibraryTests","timestamp":1398211102.803775,"event":"begin-test","methodName":"testWillPass"} +{"result":"success","test":"-[TestProject_LibraryTests testWillPass]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testWillPass","totalDuration":"0","timestamp":1398211102.803789} +{"bundleName":"TestProject-LibraryTests.octest","succeeded":true,"event":"end-ocunit","message":null,"targetName":"TestProject-LibraryTests","testType":"logic-test","sdkName":"iphonesimulator6.0","timestamp":1398211102.803806} +{"scheme":"TestProject-Library","succeeded":true,"event":"end-action","workspace":null,"project":"xctool-tests\/TestData\/TestProject-Library\/TestProject-Library.xcodeproj","duration":0.08434861100249691,"timestamp":1398211102.803864,"name":"run-tests"} \ No newline at end of file diff --git a/xctool/xctool/RunTestsAction.h b/xctool/xctool/RunTestsAction.h index e4f4e6b5d..34cc07038 100644 --- a/xctool/xctool/RunTestsAction.h +++ b/xctool/xctool/RunTestsAction.h @@ -60,6 +60,7 @@ typedef enum { @property (nonatomic, assign) BOOL freshInstall; @property (nonatomic, assign) BOOL parallelize; @property (nonatomic, assign) BOOL failOnEmptyTestBundles; +@property (nonatomic, assign) BOOL listTestsOnly; @property (nonatomic, assign) cpu_type_t cpuType; @property (nonatomic, copy) NSString *simulatorType; @property (nonatomic, copy) NSString *testSDK; diff --git a/xctool/xctool/RunTestsAction.m b/xctool/xctool/RunTestsAction.m index 85e447749..e929e6a72 100644 --- a/xctool/xctool/RunTestsAction.m +++ b/xctool/xctool/RunTestsAction.m @@ -157,6 +157,10 @@ + (NSArray *)options aliases:nil description:@"Fail when an empty test bundle was run." setFlag:@selector(setFailOnEmptyTestBundles:)], + [Action actionOptionWithName:@"listTestsOnly" + aliases:nil + description:@"Skip actual test running and list them only." + setFlag:@selector(setListTestsOnly:)], ]; } @@ -509,6 +513,10 @@ - (BOOL)runTestables:(NSArray *)testables ReportStatusMessageEnd(options.reporters, REPORTER_MESSAGE_INFO, @"Collecting info for testables..."); + if (_listTestsOnly) { + return [self listTestsInTestableExecutionInfos:testableExecutionInfos options:options]; + } + for (TestableExecutionInfo *info in testableExecutionInfos) { if (info.buildSettingsError) { TestableBlock block = [self blockToAdvertiseMessage:info.buildSettingsError @@ -653,4 +661,41 @@ - (BOOL)runTestables:(NSArray *)testables return succeeded; } +- (BOOL)listTestsInTestableExecutionInfos:(NSArray *)testableExecutionInfos + options:(Options *)options +{ + for (TestableExecutionInfo *testableExecutionInfo in testableExecutionInfos) { + PublishEventToReporters(options.reporters, + [[self class] eventForBeginOCUnitFromTestableExecutionInfo:testableExecutionInfo]); + + for (NSString *testCase in testableExecutionInfo.testCases) { + NSArray *components = [testCase componentsSeparatedByString:@"/"]; + NSString *className = components[0]; + NSString *methodName = components[1]; + NSString *testName = [NSString stringWithFormat:@"-[%@ %@]", className, methodName]; + NSDictionary *beginTestEvent = @{kReporter_BeginTest_TestKey: testName, + kReporter_BeginTest_ClassNameKey: className, + kReporter_BeginTest_MethodNameKey: methodName}; + PublishEventToReporters(options.reporters, + EventDictionaryWithNameAndContent(kReporter_Events_BeginTest, beginTestEvent)); + + + NSDictionary *endTestEvent = @{kReporter_EndTest_TestKey: testName, + kReporter_EndTest_ClassNameKey: className, + kReporter_EndTest_MethodNameKey: methodName, + kReporter_EndTest_SucceededKey: @"1", + kReporter_EndTest_ResultKey: @"success", + kReporter_EndTest_TotalDurationKey: @"0"}; + PublishEventToReporters(options.reporters, + EventDictionaryWithNameAndContent(kReporter_Events_EndTest, endTestEvent)); + } + + PublishEventToReporters(options.reporters, + [[self class] eventForEndOCUnitFromTestableExecutionInfo:testableExecutionInfo + succeeded:YES + failureReason:nil]); + } + return YES; +} + @end diff --git a/xctool/xctool/TestAction.m b/xctool/xctool/TestAction.m index 455e4ba9e..f5a474975 100644 --- a/xctool/xctool/TestAction.m +++ b/xctool/xctool/TestAction.m @@ -95,6 +95,10 @@ + (NSArray *)options description:@"Set simulator type (either iphone or ipad)" paramName:@"SIMULATOR" mapTo:@selector(setSimulatorType:)], + [Action actionOptionWithName:@"listTestsOnly" + aliases:nil + description:@"Skip actual test running and list them only." + setFlag:@selector(setListTestsOnly:)], ]; } @@ -173,6 +177,11 @@ - (void)setFailOnEmptyTestBundles:(BOOL)failOnEmptyTestBundles [_runTestsAction setFailOnEmptyTestBundles:failOnEmptyTestBundles]; } +- (void)setListTestsOnly:(BOOL)listTestsOnly +{ + [_runTestsAction setListTestsOnly:listTestsOnly]; +} + - (void)addOnly:(NSString *)argument { // build-tests takes only a target argument, where run-tests takes Target:Class/method.