Add new flag to run-tests/test action: -listTestsOnly

xctool/xctool-tests/RunTestsActionTests.m

@@ -32,6 +32,29 @@
 #import "XCToolUtil.h"
 #import "XcodeSubjectInfo.h"
 
+static BOOL areEqualJsonOutputsIgnoringKeys(NSString *output1, NSString *output2, NSArray *keys)
+{
+ NSArray *output1Array = [[output1 stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] componentsSeparatedByString:@"\n"];
+ NSArray *output2Array = [[output2 stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] componentsSeparatedByString:@"\n"];
+ if ([output1Array count] != [output2Array count]) {
+ return NO;
+ }
+
+ for (int i=0; i<[output1Array count]; i++) {
+ NSMutableDictionary *dict1 = [[NSJSONSerialization JSONObjectWithData:[output1Array[i] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] mutableCopy];
+ NSMutableDictionary *dict2 = [[NSJSONSerialization JSONObjectWithData:[output2Array[i] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] mutableCopy];
+ for (NSString *key in keys) {
+ [dict1 removeObjectForKey:key];
+ [dict2 removeObjectForKey:key];
+ }
+ if (![dict1 isEqual:dict2]) {
+ return NO;
+ }
+ }
+
+ return YES;
+}
+
 @interface RunTestsActionTests : SenTestCase
 @end
 
@@ -274,6 +297,48 @@ - (void)testRunTestsAction
  }];
 }
 
+- (void)testRunTestsActionWithListTestsOnlyOption
+{
+ NSArray *testList = @[@"TestProject_LibraryTests/testOutputMerging",
+ @"TestProject_LibraryTests/testPrintSDK",
+ @"TestProject_LibraryTests/testStream",
+ @"TestProject_LibraryTests/testWillFail",
+ @"TestProject_LibraryTests/testWillPass"];
+
+ [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+ [[FakeTaskManager sharedManager] addLaunchHandlerBlocks:@[
+ // Make sure -showBuildSettings returns some data
+ [LaunchHandlers handlerForShowBuildSettingsWithProject:TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj"
+ scheme:@"TestProject-Library"
+ settingsPath:TEST_DATA @"TestProject-Library-showBuildSettings.txt"],
+ // We're going to call -showBuildSettings on the test target.
+ [LaunchHandlers handlerForShowBuildSettingsWithProject:TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj"
+ target:@"TestProject-LibraryTests"
+ settingsPath:TEST_DATA @"TestProject-Library-TestProject-LibraryTests-showBuildSettings.txt"
+ hide:NO],
+ [LaunchHandlers handlerForOtestQueryReturningTestList:testList],
+ ]];
+
+ XCTool *tool = [[[XCTool alloc] init] autorelease];
+
+ tool.arguments = @[@"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-configuration", @"Debug",
+ @"-sdk", @"iphonesimulator6.0",
+ @"run-tests",
+ @"listTestsOnly",
+ @"-reporter", @"json-stream"
+ ];
+
+ NSDictionary *result = [TestUtil runWithFakeStreams:tool];
+ NSString *listTestsOnlyOutput = [NSString stringWithContentsOfFile:TEST_DATA @"TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt"
+ encoding:NSUTF8StringEncoding
+ error:nil];
+ NSString *stdoutString = result[@"stdout"];
+ assertThatBool(areEqualJsonOutputsIgnoringKeys(stdoutString, listTestsOnlyOutput, @[@"timestamp", @"duration"]), equalToBool(YES));
+ }];
+}
+
 - (void)testCanRunTestsAgainstDifferentTestSDK
 {
  NSArray *testList = @[@"TestProject_LibraryTests/testBacktraceOutputIsCaptured",

xctool/xctool-tests/TestData/TestProject-Library-TestProject-LibraryTests-run-test-results-listtestonly.txt

@@ -0,0 +1,18 @@
+{"message":"Loading settings for scheme 'TestProject-Library' ...","timestamp":1398211102.696059,"level":"Info","event":"begin-status"}
+{"message":"Loading settings for scheme 'TestProject-Library' ...","timestamp":1398211102.719442,"level":"Info","event":"end-status"}
+{"scheme":"TestProject-Library","workspace":null,"timestamp":1398211102.719507,"event":"begin-action","project":"xctool-tests\/TestData\/TestProject-Library\/TestProject-Library.xcodeproj","name":"run-tests"}
+{"message":"Collecting info for testables...","timestamp":1398211102.719552,"level":"Info","event":"begin-status"}
+{"message":"Collecting info for testables...","timestamp":1398211102.803515,"level":"Info","event":"end-status"}
+{"bundleName":"TestProject-LibraryTests.octest","sdkName":"iphonesimulator6.0","timestamp":1398211102.803576,"event":"begin-ocunit","targetName":"TestProject-LibraryTests","testType":"logic-test"}
+{"test":"-[TestProject_LibraryTests testOutputMerging]","className":"TestProject_LibraryTests","timestamp":1398211102.803618,"event":"begin-test","methodName":"testOutputMerging"}
+{"result":"success","test":"-[TestProject_LibraryTests testOutputMerging]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testOutputMerging","totalDuration":"0","timestamp":1398211102.803637}
+{"test":"-[TestProject_LibraryTests testPrintSDK]","className":"TestProject_LibraryTests","timestamp":1398211102.803671,"event":"begin-test","methodName":"testPrintSDK"}
+{"result":"success","test":"-[TestProject_LibraryTests testPrintSDK]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testPrintSDK","totalDuration":"0","timestamp":1398211102.803688}
+{"test":"-[TestProject_LibraryTests testStream]","className":"TestProject_LibraryTests","timestamp":1398211102.803714,"event":"begin-test","methodName":"testStream"}
+{"result":"success","test":"-[TestProject_LibraryTests testStream]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testStream","totalDuration":"0","timestamp":1398211102.803728}
+{"test":"-[TestProject_LibraryTests testWillFail]","className":"TestProject_LibraryTests","timestamp":1398211102.803743,"event":"begin-test","methodName":"testWillFail"}
+{"result":"success","test":"-[TestProject_LibraryTests testWillFail]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testWillFail","totalDuration":"0","timestamp":1398211102.803755}
+{"test":"-[TestProject_LibraryTests testWillPass]","className":"TestProject_LibraryTests","timestamp":1398211102.803775,"event":"begin-test","methodName":"testWillPass"}
+{"result":"success","test":"-[TestProject_LibraryTests testWillPass]","className":"TestProject_LibraryTests","succeeded":"1","event":"end-test","methodName":"testWillPass","totalDuration":"0","timestamp":1398211102.803789}
+{"bundleName":"TestProject-LibraryTests.octest","succeeded":true,"event":"end-ocunit","message":null,"targetName":"TestProject-LibraryTests","testType":"logic-test","sdkName":"iphonesimulator6.0","timestamp":1398211102.803806}
+{"scheme":"TestProject-Library","succeeded":true,"event":"end-action","workspace":null,"project":"xctool-tests\/TestData\/TestProject-Library\/TestProject-Library.xcodeproj","duration":0.08434861100249691,"timestamp":1398211102.803864,"name":"run-tests"}

xctool/xctool/RunTestsAction.h

@@ -60,6 +60,7 @@ typedef enum {
 @property (nonatomic, assign) BOOL freshInstall;
 @property (nonatomic, assign) BOOL parallelize;
 @property (nonatomic, assign) BOOL failOnEmptyTestBundles;
+@property (nonatomic, assign) BOOL listTestsOnly;
 @property (nonatomic, assign) cpu_type_t cpuType;
 @property (nonatomic, copy) NSString *simulatorType;
 @property (nonatomic, copy) NSString *testSDK;

xctool/xctool/RunTestsAction.m

@@ -157,6 +157,10 @@ + (NSArray *)options
  aliases:nil
  description:@"Fail when an empty test bundle was run."
  setFlag:@selector(setFailOnEmptyTestBundles:)],
+ [Action actionOptionWithName:@"listTestsOnly"
+ aliases:nil
+ description:@"Skip actual test running and list them only."
+ setFlag:@selector(setListTestsOnly:)],
  ];
 }
 
@@ -509,6 +513,10 @@ - (BOOL)runTestables:(NSArray *)testables
  ReportStatusMessageEnd(options.reporters, REPORTER_MESSAGE_INFO,
  @"Collecting info for testables...");
 
+ if (_listTestsOnly) {
+ return [self listTestsInTestableExecutionInfos:testableExecutionInfos options:options];
+ }
+
  for (TestableExecutionInfo *info in testableExecutionInfos) {
  if (info.buildSettingsError) {
  TestableBlock block = [self blockToAdvertiseMessage:info.buildSettingsError
@@ -653,4 +661,41 @@ - (BOOL)runTestables:(NSArray *)testables
  return succeeded;
 }
 
+- (BOOL)listTestsInTestableExecutionInfos:(NSArray *)testableExecutionInfos
+ options:(Options *)options
+{
+ for (TestableExecutionInfo *testableExecutionInfo in testableExecutionInfos) {
+ PublishEventToReporters(options.reporters,
+ [[self class] eventForBeginOCUnitFromTestableExecutionInfo:testableExecutionInfo]);
+
+ for (NSString *testCase in testableExecutionInfo.testCases) {
+ NSArray *components = [testCase componentsSeparatedByString:@"/"];
+ NSString *className = components[0];
+ NSString *methodName = components[1];
+ NSString *testName = [NSString stringWithFormat:@"-[%@ %@]", className, methodName];
+ NSDictionary *beginTestEvent = @{kReporter_BeginTest_TestKey: testName,
+ kReporter_BeginTest_ClassNameKey: className,
+ kReporter_BeginTest_MethodNameKey: methodName};
+ PublishEventToReporters(options.reporters,
+ EventDictionaryWithNameAndContent(kReporter_Events_BeginTest, beginTestEvent));
+
+
+ NSDictionary *endTestEvent = @{kReporter_EndTest_TestKey: testName,
+ kReporter_EndTest_ClassNameKey: className,
+ kReporter_EndTest_MethodNameKey: methodName,
+ kReporter_EndTest_SucceededKey: @"1",
+ kReporter_EndTest_ResultKey: @"success",
+ kReporter_EndTest_TotalDurationKey: @"0"};
+ PublishEventToReporters(options.reporters,
+ EventDictionaryWithNameAndContent(kReporter_Events_EndTest, endTestEvent));
+ }
+
+ PublishEventToReporters(options.reporters,
+ [[self class] eventForEndOCUnitFromTestableExecutionInfo:testableExecutionInfo
+ succeeded:YES
+ failureReason:nil]);
+ }
+ return YES;
+}
+
 @end

xctool/xctool/TestAction.m

@@ -95,6 +95,10 @@ + (NSArray *)options
  description:@"Set simulator type (either iphone or ipad)"
  paramName:@"SIMULATOR"
  mapTo:@selector(setSimulatorType:)],
+ [Action actionOptionWithName:@"listTestsOnly"
+ aliases:nil
+ description:@"Skip actual test running and list them only."
+ setFlag:@selector(setListTestsOnly:)],
  ];
 }
 
@@ -173,6 +177,11 @@ - (void)setFailOnEmptyTestBundles:(BOOL)failOnEmptyTestBundles
  [_runTestsAction setFailOnEmptyTestBundles:failOnEmptyTestBundles];
 }
 
+- (void)setListTestsOnly:(BOOL)listTestsOnly
+{
+ [_runTestsAction setListTestsOnly:listTestsOnly];
+}
+
 - (void)addOnly:(NSString *)argument
 {
  // build-tests takes only a target argument, where run-tests takes Target:Class/method.