Add an omit option for tests

xctool/xctool-tests/BuildTestsActionTests.m

@@ -48,6 +48,23 @@ - (void)setUp
  [super setUp];
 }
 
+- (void)testOnlyListAndOmitListCannotBothBeSpecified
+{
+ [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-sdk", @"iphonesimulator6.1",
+ @"build-tests",
+ @"-only", @"TestProject-LibraryTests",
+ @"-omit", @"TestProject-LibraryTests",
+ ]]
+ assertOptionsFailToValidateWithError:
+ @"build-tests: -only and -omit cannot both be specified."
+ withBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+}
+
 - (void)testOnlyListIsCollected
 {
  Options *options = [[Options optionsFrom:@[
@@ -62,6 +79,19 @@ - (void)testOnlyListIsCollected
  assertThat((action.onlyList), equalTo(@[@"TestProject-LibraryTests"]));
 }
 
+- (void)testOmitListIsCollected
+{
+ Options *options = [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-sdk", @"iphonesimulator6.1",
+ @"build-tests", @"-omit", @"TestProject-LibraryTests",
+ ]] assertOptionsValidateWithBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+ BuildTestsAction *action = options.actions[0];
+ assertThat((action.omitList), equalTo(@[@"TestProject-LibraryTests"]));
+}
 
 - (void)testSkipDependenciesIsCollected
 {

xctool/xctool-tests/RunTestsActionTests.m

@@ -84,6 +84,23 @@ - (void)testTestSDKIsCollected
  assertThat((action.testSDK), equalTo(@"iphonesimulator6.0"));
 }
 
+- (void)testOnlyListAndOmitListCannotBothBeSpecified
+{
+ [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-sdk", @"iphonesimulator6.1",
+ @"run-tests",
+ @"-only", @"TestProject-LibraryTests",
+ @"-omit", @"TestProject-LibraryTests",
+ ]]
+ assertOptionsFailToValidateWithError:
+ @"run-tests: -only and -omit cannot both be specified."
+ withBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+}
+
 - (void)testOnlyListIsCollected
 {
  Options *options = [[Options optionsFrom:@[
@@ -98,6 +115,20 @@ - (void)testOnlyListIsCollected
  assertThat((action.onlyList), equalTo(@[@"TestProject-LibraryTests"]));
 }
 
+- (void)testOmitListIsCollected
+{
+ Options *options = [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-sdk", @"iphonesimulator6.1",
+ @"run-tests", @"-omit", @"TestProject-LibraryTests",
+ ]] assertOptionsValidateWithBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+ RunTestsAction *action = options.actions[0];
+ assertThat((action.omitList), equalTo(@[@"TestProject-LibraryTests"]));
+}
+
 - (void)testOnlyListRequiresValidTarget
 {
  [[Options optionsFrom:@[
@@ -529,7 +560,7 @@ - (void)testCanRunTestsAgainstDifferentTestSDK
  }];
 }
 
-- (void)testCanSelectSpecificTestClassOrTestMethodWithOnly
+- (void)testCanSelectSpecificTestClassOrTestMethodsWithOnlyAndOmit
 {
  if (ToolchainIsXcode7OrBetter()) {
  // octest isn't supported in Xcode 7
@@ -545,7 +576,7 @@ - (void)testCanSelectSpecificTestClassOrTestMethodWithOnly
  @"SomeTests/testWillFail",
  @"SomeTests/testWillPass"];
 
- void (^runWithOnlyArgumentAndExpectSenTestToBe)(NSString *, NSString *) = ^(NSString *onlyArgument, NSString *expectedSenTest) {
+ void (^runWithArguments)(NSString *, NSArray *, BOOL) = ^(NSString *argument, NSArray *values, BOOL skipTarget) {
  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
  [[FakeTaskManager sharedManager] addLaunchHandlerBlocks:@[
  // Make sure -showBuildSettings returns some data
@@ -570,58 +601,62 @@ - (void)testCanSelectSpecificTestClassOrTestMethodWithOnly
  ]];
 
  XCTool *tool = [[XCTool alloc] init];
-
- tool.arguments = @[@"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
- @"-scheme", @"TestProject-Library",
- @"-configuration", @"Debug",
- @"-sdk", @"iphonesimulator6.0",
- @"-destination", @"arch=i386",
- @"run-tests", @"-only", onlyArgument,
- @"-reporter", @"plain",
- ];
+ NSMutableArray *args = [@[@"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-configuration", @"Debug",
+ @"-sdk", @"iphonesimulator6.0",
+ @"-destination", @"arch=i386",
+ @"run-tests"] mutableCopy];
+ for (NSString *value in values) {
+ [args addObject:argument];
+ [args addObject:value];
+ }
+ [args addObjectsFromArray:@[@"-reporter", @"plain"]];
+ tool.arguments = args;
 
  [TestUtil runWithFakeStreams:tool];
 
  NSArray *launchedTasks = [[FakeTaskManager sharedManager] launchedTasks];
- assertThatInteger([launchedTasks count], equalToInteger(2));
- NSArray *arguments = [launchedTasks[1] arguments];
- assertThat(arguments, containsArray(@[
- @"-NSTreatUnknownArgumentsAsOpen", @"NO",
- @"-ApplePersistenceIgnoreState", @"YES",
- @"-SenTestInvertScope", @"YES"]));
- assertThat(arguments, containsArray(@[@"-OTEST_TESTLIST_FILE"]));
- assertThat(arguments, containsArray(@[
- @"-OTEST_FILTER_TEST_ARGS_KEY", @"SenTest",
- @"-SenTest", @"XCTOOL_FAKE_LIST_OF_TESTS",
- ]));
-
- assertThat(arguments, containsArray(@[
- @"/Users/nekto/Library/Developer/Xcode/DerivedData/TestProject-Library-frruszglismbfoceinskphldzhci/Build/Products/Debug-iphonesimulator/TestProject-LibraryTests.octest",
- ]));
+ if (skipTarget) {
+ assertThatInteger([launchedTasks count], equalToInteger(0));
+ } else {
+ assertThatInteger([launchedTasks count], equalToInteger(2));
+ NSArray *arguments = [launchedTasks[1] arguments];
+ assertThat(arguments, containsArray(@[
+ @"-NSTreatUnknownArgumentsAsOpen", @"NO",
+ @"-ApplePersistenceIgnoreState", @"YES",
+ @"-SenTestInvertScope", @"YES"]));
+ assertThat(arguments, containsArray(@[@"-OTEST_TESTLIST_FILE"]));
+ assertThat(arguments, containsArray(@[
+ @"-OTEST_FILTER_TEST_ARGS_KEY", @"SenTest",
+ @"-SenTest", @"XCTOOL_FAKE_LIST_OF_TESTS",
+ ]));
+
+ assertThat(arguments, containsArray(@[
+ @"/Users/nekto/Library/Developer/Xcode/DerivedData/TestProject-Library-frruszglismbfoceinskphldzhci/Build/Products/Debug-iphonesimulator/TestProject-LibraryTests.octest",
+ ]));
+ }
  }];
  };
 
- runWithOnlyArgumentAndExpectSenTestToBe(@"TestProject-LibraryTests:SomeTests/testOutputMerging",
- @"OtherTests/testSomething,"
- @"SomeTests/testBacktraceOutputIsCaptured,"
- @"SomeTests/testPrintSDK,"
- @"SomeTests/testStream,"
- @"SomeTests/testWillFail,"
- @"SomeTests/testWillPass");
- runWithOnlyArgumentAndExpectSenTestToBe(@"TestProject-LibraryTests:SomeTests/testWillPass",
- @"OtherTests/testSomething,"
- @"SomeTests/testBacktraceOutputIsCaptured,"
- @"SomeTests/testOutputMerging,"
- @"SomeTests/testPrintSDK,"
- @"SomeTests/testStream,"
- @"SomeTests/testWillFail");
- runWithOnlyArgumentAndExpectSenTestToBe(@"TestProject-LibraryTests:SomeTests/testWillPass,OtherTests/testSomething",
- // The ordering will be alphabetized.
- @"SomeTests/testBacktraceOutputIsCaptured,"
- @"SomeTests/testOutputMerging,"
- @"SomeTests/testPrintSDK,"
- @"SomeTests/testStream,"
- @"SomeTests/testWillFail");
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests:SomeTests/testOutputMerging"], NO);
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests:SomeTests/testWillPass"], NO);
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests:SomeTests/testWillPass,OtherTests/testSomething"], NO);
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests:SomeTests/testWillPass",
+ @"TestProject-LibraryTests:SomeTests/testWillFail"], NO);
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests",
+ @"TestProject-LibraryTests:SomeTests/testWillFail"], NO);
+ runWithArguments(@"-only", @[@"TestProject-LibraryTests:SomeTests/testWillPass",
+ @"TestProject-LibraryTests"], NO);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests:SomeTests/testOutputMerging"], NO);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests:SomeTests/testWillPass"], NO);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests:SomeTests/testWillPass,OtherTests/testSomething"], NO);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests:SomeTests/testWillPass",
+ @"TestProject-LibraryTests:SomeTests/testWillFail,SomeTests/testOutputMerging"], NO);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests",
+ @"TestProject-LibraryTests:SomeTests/testWillFail,SomeTests/testOutputMerging"], YES);
+ runWithArguments(@"-omit", @[@"TestProject-LibraryTests:SomeTests/testWillPass",
+ @"TestProject-LibraryTests"], YES);
 }
 
 /**

xctool/xctool-tests/TestActionTests.m

@@ -54,6 +54,20 @@ - (void)testOnlyListIsCollected
  assertThat(([action onlyList]), equalTo(@[@"TestProject-LibraryTests"]));
 }
 
+- (void)testOmitListIsCollected
+{
+ Options *options = [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"-sdk", @"iphonesimulator6.1",
+ @"test", @"-omit", @"TestProject-LibraryTests",
+ ]] assertOptionsValidateWithBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+ TestAction *action = options.actions[0];
+ assertThat(([action omitList]), equalTo(@[@"TestProject-LibraryTests"]));
+}
+
 - (void)testOnlyListRequiresValidTarget
 {
  [[Options optionsFrom:@[
@@ -97,4 +111,17 @@ - (void)testOnlyParsing
  assertThat([options.actions[0] runTestsAction].onlyList, equalTo(@[@"TestProject-LibraryTests:ClassName/methodName"]));
 }
 
+- (void)testOmitParsing
+{
+ Options *options = [[Options optionsFrom:@[
+ @"-project", TEST_DATA @"TestProject-Library/TestProject-Library.xcodeproj",
+ @"-scheme", @"TestProject-Library",
+ @"test", @"-omit", @"TestProject-LibraryTests:ClassName/methodName"
+ ]] assertOptionsValidateWithBuildSettingsFromFile:
+ TEST_DATA @"TestProject-Library-TestProject-Library-showBuildSettings.txt"
+ ];
+ assertThat([options.actions[0] buildTestsAction].omitList, equalTo(@[@"TestProject-LibraryTests"]));
+ assertThat([options.actions[0] runTestsAction].omitList, equalTo(@[@"TestProject-LibraryTests:ClassName/methodName"]));
+}
+
 @end

xctool/xctool/BuildTestsAction.h

@@ -19,6 +19,7 @@
 @interface BuildTestsAction : Action
 
 @property (nonatomic, strong) NSMutableArray *onlyList;
+@property (nonatomic, strong) NSMutableArray *omitList;
 @property (nonatomic, assign) BOOL skipDependencies;
 
 + (BOOL)buildWorkspace:(NSString *)path

xctool/xctool/BuildTestsAction.m

@@ -35,11 +35,16 @@ + (NSString *)name
 + (NSArray *)options
 {
  return @[
- [Action actionOptionWithName:@"only"
- aliases:nil
- description:@"build only a specific test TARGET"
- paramName:@"TARGET"
- mapTo:@selector(addOnly:)],
+ [Action actionOptionWithName:@"only"
+ aliases:nil
+ description:@"build only a specific test TARGET"
+ paramName:@"TARGET"
+ mapTo:@selector(addOnly:)],
+ [Action actionOptionWithName:@"omit"
+ aliases:nil
+ description:@"omit building a specific test TARGET"
+ paramName:@"TARGET"
+ mapTo:@selector(addOmit:)],
  [Action actionOptionWithName:@"skip-deps"
  aliases:nil
  description:@"Only build the target, not its dependencies"
@@ -139,6 +144,7 @@ - (instancetype)init
 {
  if (self = [super init]) {
  _onlyList = [[NSMutableArray alloc] init];
+ _omitList = [[NSMutableArray alloc] init];
  }
  return self;
 }
@@ -149,10 +155,19 @@ - (void)addOnly:(NSString *)argument
  [_onlyList addObject:argument];
 }
 
+- (void)addOmit:(NSString *)argument
+{
+ [_omitList addObject:argument];
+}
+
 - (BOOL)validateWithOptions:(Options *)options
  xcodeSubjectInfo:(XcodeSubjectInfo *)xcodeSubjectInfo
  errorMessage:(NSString **)errorMessage
 {
+ if (_onlyList.count > 0 && _omitList.count > 0) {
+ *errorMessage = @"build-tests: -only and -omit cannot both be specified.";
+ return NO;
+ }
  for (NSString *target in _onlyList) {
  if ([xcodeSubjectInfo testableWithTarget:target] == nil) {
  *errorMessage = [NSString stringWithFormat:@"build-tests: '%@' is not a testing target in this scheme.", target];
@@ -164,19 +179,22 @@ - (BOOL)validateWithOptions:(Options *)options
 }
 
 - (NSMutableArray *)buildableList:(NSArray *)buildableList
- matchingTargets:(NSArray *)targets
+ matchingTargets:(NSArray *)onlyList
+ excludingTargets:(NSArray *)omitList
 {
  NSMutableArray *result = [NSMutableArray array];
 
  for (Buildable *buildable in buildableList) {
  BOOL add;
- if (targets.count > 0 && [[buildable.executable pathExtension] isEqualToString:@"octest"]) {
+ if (onlyList.count > 0 && [[buildable.executable pathExtension] isEqualToString:@"octest"]) {
  // If we're filtering by target, only add targets that match.
- add = [targets containsObject:buildable.target];
+ add = [onlyList containsObject:buildable.target];
  } else if (_skipDependencies) {
  add = NO;
  } else {
- add = !([buildable isKindOfClass:[Testable class]] && [(Testable *)buildable skipped]);
+ add = !([buildable isKindOfClass:[Testable class]] &&
+ ([(Testable *)buildable skipped] ||
+ [omitList containsObject:buildable.target]));
  }
  if (add) {
  [result addObject:buildable];
@@ -189,7 +207,8 @@ - (NSMutableArray *)buildableList:(NSArray *)buildableList
 - (BOOL)performActionWithOptions:(Options *)options xcodeSubjectInfo:(XcodeSubjectInfo *)xcodeSubjectInfo
 {
  NSArray *buildableList = [self buildableList:[xcodeSubjectInfo testablesAndBuildablesForTest]
- matchingTargets:_onlyList];
+ matchingTargets:_onlyList
+ excludingTargets:_omitList];
  if (!buildableList.count) {
  return YES;
  }

xctool/xctool/RunTestsAction.h

@@ -60,6 +60,7 @@ typedef NS_ENUM(NSInteger, BucketBy) {
 @property (nonatomic, assign) BOOL listTestsOnly;
 @property (nonatomic, copy) NSString *testSDK;
 @property (nonatomic, strong) NSMutableArray *onlyList;
+@property (nonatomic, strong) NSMutableArray *omitList;
 @property (nonatomic, strong) NSMutableArray *logicTests;
 @property (nonatomic, strong) NSMutableDictionary *appTests;
 @property (nonatomic, copy) NSString *targetedDeviceFamily;