#357 Defaulting to using CMA except for servers

Moved prepareCommand into a new listener class

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
 version=3.15.develop
-mlJavaclientUtilVersion=3.12.0
+mlJavaclientUtilVersion=3.13.develop
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -73,7 +73,7 @@ public class AppConfig {
     private boolean catchDeployExceptions = false;
     private boolean catchUndeployExceptions = false;
 
-    private CmaConfig cmaConfig = new CmaConfig();
+    private CmaConfig cmaConfig;
     private boolean mergeResources = true;
 
     private boolean addHostNameTokens = false;
@@ -251,6 +251,11 @@ public AppConfig(File projectDir) {
 		dataConfig = new DataConfig(projectDir);
 		pluginConfig = new PluginConfig(projectDir);
 
+		// As of 3.15.0, defaulting everything except servers to use CMA. Changes to servers, such as changing group or
+	    // port number, cause conflicts with CMA.
+		cmaConfig = new CmaConfig(true);
+		cmaConfig.setDeployServers(false);
+
 		modulePaths = new ArrayList<>();
 		String path = projectDir != null ? new File(projectDir, DEFAULT_MODULES_PATH).getAbsolutePath() : DEFAULT_MODULES_PATH;
 		modulePaths.add(path);

src/main/java/com/marklogic/appdeployer/CmaConfig.java

@@ -13,6 +13,15 @@ public class CmaConfig {
 	private boolean deployServers;
 	private boolean deployUsers;
 
+	public CmaConfig() {
+	}
+
+	public CmaConfig(boolean enableAll) {
+		if (enableAll) {
+			enableAll();
+		}
+	}
+
 	public void enableAll() {
 		setCombineRequests(true);
 		setDeployAmps(true);

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -63,22 +63,24 @@ public void initialize() {
 			config.setMergeResources(Boolean.parseBoolean(prop));
 		});
 
-		propertyConsumerMap.put("mlOptimizeWithCma", (config, prop) -> {
-			logger.info("mlOptimizeWithCma is DEPRECATED; please use a property specific to the resource that you want to deploy with CMA");
-			// mlOptimizeWithCma was deprecated in 3.11; it was only used for deploying forests, so if the
-			// property is still used, the client in theory expects forests to still be deployed with CMA
-			config.getCmaConfig().setDeployForests(true);
-		});
-
 		final String cmaMessage = " with the Configuration Management API (CMA): ";
 
 		propertyConsumerMap.put("mlDeployWithCma", (config, prop) -> {
 			logger.info("Deploy all supported resources and combine requests" + cmaMessage + prop);
 			if (Boolean.parseBoolean(prop)) {
 				config.getCmaConfig().enableAll();
+			} else {
+				config.setCmaConfig(new CmaConfig());
 			}
 		});
 
+		propertyConsumerMap.put("mlOptimizeWithCma", (config, prop) -> {
+			logger.info("mlOptimizeWithCma is DEPRECATED; please use a property specific to the resource that you want to deploy with CMA");
+			// mlOptimizeWithCma was deprecated in 3.11; it was only used for deploying forests, so if the
+			// property is still used, the client in theory expects forests to still be deployed with CMA
+			config.getCmaConfig().setDeployForests(true);
+		});
+
 		propertyConsumerMap.put("mlCombineCmaRequests", (config, prop) -> {
 			logger.info("Combine requests" + cmaMessage + prop);
 			config.getCmaConfig().setCombineRequests(Boolean.parseBoolean(prop));

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -96,6 +96,7 @@ public class DeployDatabaseCommand extends AbstractCommand implements UndoableCo
     public DeployDatabaseCommand() {
 	    setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_DATABASES);
         setUndoSortOrder(SortOrderConstants.DELETE_OTHER_DATABASES);
+        setResourceClassType(Database.class);
     }
 
     public DeployDatabaseCommand(File databaseFile) {

src/main/java/com/marklogic/appdeployer/command/databases/DeployOtherDatabasesCommand.java

@@ -51,6 +51,7 @@ public DeployOtherDatabasesCommand() {
 		setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_DATABASES);
 		setUndoSortOrder(SortOrderConstants.DELETE_OTHER_DATABASES);
 		initializeDefaultDatabasesToNotUndeploy();
+		setResourceClassType(Database.class);
 	}
 
 	public DeployOtherDatabasesCommand(int forestsPerHost) {

src/main/java/com/marklogic/appdeployer/impl/AbstractAppDeployer.java

@@ -3,7 +3,6 @@
 import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.AppDeployer;
 import com.marklogic.appdeployer.ConfigDir;
-import com.marklogic.appdeployer.command.AbstractCommand;
 import com.marklogic.appdeployer.command.Command;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.UndoableCommand;
@@ -15,7 +14,6 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.regex.Pattern;
 
 /**
  * Abstract base class that just needs the subclass to define the list of Command instances to use. Handles executing
@@ -41,6 +39,7 @@ public AbstractAppDeployer(ManageClient manageClient, AdminManager adminManager)
 
 		this.deployerListeners = new ArrayList<>();
 		this.deployerListeners.add(new AddHostNameTokensDeployerListener());
+		this.deployerListeners.add(new PrepareCommandListener());
 		this.deployerListeners.add(new CmaDeployerListener());
 	}
 
@@ -72,50 +71,25 @@ public void deploy(AppConfig appConfig) {
 
 		deployerListeners.forEach(listener -> listener.beforeCommandsExecuted(deploymentContext));
 
+		boolean catchExceptions = appConfig.isCatchDeployExceptions();
+
 		int commandCount = commands.size();
 		for (int i = 0; i < commandCount; i++) {
 			Command command = commands.get(i);
 			final List<Command> remainingCommands = commands.subList(i + 1, commandCount);
 			String name = command.getClass().getName();
 
 			logger.info(format("Executing command [%s] with sort order [%d]", name, command.getExecuteSortOrder()));
-			prepareCommand(command, context);
-			deployerListeners.forEach(listener -> listener.beforeCommandExecuted(command, deploymentContext, remainingCommands));
+			invokeListenersBeforeCommandExecuted(context, command, deploymentContext, remainingCommands, catchExceptions);
 			long start = System.currentTimeMillis();
 			executeCommand(command, context);
 			logger.info(format("Finished executing command [%s] in %dms\n", name, (System.currentTimeMillis() - start)));
-			deployerListeners.forEach(listener -> listener.afterCommandExecuted(command, deploymentContext, remainingCommands));
+			invokeListenersAfterCommandExecuted(context, command, deploymentContext, remainingCommands, catchExceptions);
 		}
 
 		logger.info(format("Deployed app %s", appConfig.getName()));
 	}
 
-	/**
-	 * Prepare the given command before either execute or undo is called on it.
-	 *
-	 * @param command
-	 * @param context
-	 */
-	protected void prepareCommand(Command command, CommandContext context) {
-		if (command instanceof AbstractCommand) {
-			AppConfig appConfig = context.getAppConfig();
-			String[] filenamesToIgnore = appConfig.getResourceFilenamesToIgnore();
-			Pattern excludePattern = appConfig.getResourceFilenamesExcludePattern();
-			Pattern includePattern = appConfig.getResourceFilenamesIncludePattern();
-
-			AbstractCommand abstractCommand = (AbstractCommand) command;
-			if (filenamesToIgnore != null) {
-				abstractCommand.setFilenamesToIgnore(filenamesToIgnore);
-			}
-			if (excludePattern != null) {
-				abstractCommand.setResourceFilenamesExcludePattern(excludePattern);
-			}
-			if (includePattern != null) {
-				abstractCommand.setResourceFilenamesIncludePattern(includePattern);
-			}
-		}
-	}
-
 	/**
 	 * Executes the command, catching an exception if desired.
 	 *
@@ -158,12 +132,22 @@ public void undeploy(AppConfig appConfig) {
 		Collections.sort(undoableCommands, new UndoComparator());
 		CommandContext context = new CommandContext(appConfig, manageClient, adminManager);
 
-		for (UndoableCommand command : undoableCommands) {
+		final DeploymentContext deploymentContext = new DeploymentContext(context, appConfig, commands);
+		deployerListeners.forEach(listener -> listener.beforeCommandsExecuted(deploymentContext));
+
+		boolean catchExceptions = appConfig.isCatchUndeployExceptions();
+
+		int commandCount = undoableCommands.size();
+		for (int i = 0; i < commandCount; i++) {
+			UndoableCommand command = undoableCommands.get(i);
+			final List<Command> remainingCommands = commands.subList(i + 1, commandCount);
+
 			String name = command.getClass().getName();
 			logger.info(format("Undoing command [%s] with sort order [%d]", name, command.getUndoSortOrder()));
-			prepareCommand(command, context);
+			invokeListenersBeforeCommandExecuted(context, command, deploymentContext, remainingCommands, catchExceptions);
 			undoCommand(command, context);
 			logger.info(format("Finished undoing command [%s]\n", name));
+			invokeListenersAfterCommandExecuted(context, command, deploymentContext, remainingCommands, catchExceptions);
 		}
 
 		logger.info(format("Undeployed app %s", appConfig.getName()));
@@ -187,6 +171,36 @@ protected void undoCommand(UndoableCommand command, CommandContext context) {
 		}
 	}
 
+	protected void invokeListenersBeforeCommandExecuted(CommandContext context, Command command, DeploymentContext deploymentContext,
+	                                                    List<Command> remainingCommands, boolean catchExceptions) {
+		deployerListeners.forEach(listener -> {
+			try {
+				listener.beforeCommandExecuted(command, deploymentContext, remainingCommands);
+			} catch (Exception ex) {
+				if (catchExceptions) {
+					logger.error(format("Listener threw exception that was caught; cause: %s", ex.getMessage()), ex);
+				} else {
+					throw ex;
+				}
+			}
+		});
+	}
+
+	protected void invokeListenersAfterCommandExecuted(CommandContext context, Command command, DeploymentContext deploymentContext,
+	                                                   List<Command> remainingCommands, boolean catchExceptions) {
+		deployerListeners.forEach(listener -> {
+			try {
+				listener.afterCommandExecuted(command, deploymentContext, remainingCommands);
+			} catch (Exception ex) {
+				if (catchExceptions) {
+					logger.error(format("Listener threw exception that was caught; cause: %s", ex.getMessage()), ex);
+				} else {
+					throw ex;
+				}
+			}
+		});
+	}
+
 	public List<DeployerListener> getDeployerListeners() {
 		return deployerListeners;
 	}

src/main/java/com/marklogic/appdeployer/impl/CmaDeployerListener.java

@@ -2,6 +2,7 @@
 
 import com.marklogic.appdeployer.command.Command;
 import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.SortOrderConstants;
 import com.marklogic.appdeployer.command.databases.DatabasePlan;
 import com.marklogic.appdeployer.command.security.DeployAmpsCommand;
 import com.marklogic.appdeployer.command.security.DeployUsersCommand;
@@ -29,11 +30,7 @@ public class CmaDeployerListener extends DeployerListenerSupport {
 
 	@Override
 	public void afterCommandExecuted(Command command, DeploymentContext context, List<Command> remainingCommands) {
-		if (
-			command instanceof DeployUsersCommand ||
-				command instanceof DeployAmpsCommand ||
-				remainingCommands.isEmpty()
-		) {
+		if (combinedRequestBeSubmitted(command, remainingCommands)) {
 			CommandContext commandContext = context.getCommandContext();
 
 			Configurations configs = commandContext.getCombinedCmaRequest();
@@ -57,4 +54,23 @@ public void afterCommandExecuted(Command command, DeploymentContext context, Lis
 			}
 		}
 	}
+
+	protected boolean combinedRequestBeSubmitted(Command command, List<Command> remainingCommands) {
+		if (command instanceof DeployUsersCommand || command instanceof DeployAmpsCommand) {
+			return true;
+		}
+
+		if (remainingCommands.isEmpty()) {
+			return true;
+		}
+
+		/**
+		 * At least for many ml-app-deployer tests, a small subset of commands are used - for example, perhaps only the
+		 * commands for deploying databases and triggers are used. To ensure that databases are still deployed, we look
+		 * at the next command to see if it's being executed after amps are deployed, as amps are the last resource
+		 * we include in a combined request. If so, then we know we need to submit the combined request.
+		 */
+		Command nextCommand = remainingCommands.get(0);
+		return nextCommand.getExecuteSortOrder() >= SortOrderConstants.DEPLOY_AMPS;
+	}
 }

src/main/java/com/marklogic/appdeployer/impl/PrepareCommandListener.java

@@ -0,0 +1,32 @@
+package com.marklogic.appdeployer.impl;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.AbstractCommand;
+import com.marklogic.appdeployer.command.Command;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class PrepareCommandListener extends DeployerListenerSupport {
+
+	@Override
+	public void beforeCommandExecuted(Command command, DeploymentContext context, List<Command> remainingCommands) {
+		if (command instanceof AbstractCommand) {
+			AppConfig appConfig = context.getAppConfig();
+			String[] filenamesToIgnore = appConfig.getResourceFilenamesToIgnore();
+			Pattern excludePattern = appConfig.getResourceFilenamesExcludePattern();
+			Pattern includePattern = appConfig.getResourceFilenamesIncludePattern();
+
+			AbstractCommand abstractCommand = (AbstractCommand) command;
+			if (filenamesToIgnore != null) {
+				abstractCommand.setFilenamesToIgnore(filenamesToIgnore);
+			}
+			if (excludePattern != null) {
+				abstractCommand.setResourceFilenamesExcludePattern(excludePattern);
+			}
+			if (includePattern != null) {
+				abstractCommand.setResourceFilenamesIncludePattern(includePattern);
+			}
+		}
+	}
+}

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -213,27 +213,27 @@ public void deployWithCma() {
 		Properties p = new Properties();
 
 		CmaConfig cmaConfig = new DefaultAppConfigFactory(new SimplePropertySource(p)).newAppConfig().getCmaConfig();
+		assertTrue(cmaConfig.isCombineRequests());
+		assertTrue(cmaConfig.isDeployAmps());
+		assertTrue(cmaConfig.isDeployDatabases());
+		assertTrue(cmaConfig.isDeployForests());
+		assertTrue(cmaConfig.isDeployPrivileges());
+		assertTrue(cmaConfig.isDeployRoles());
+		assertFalse(cmaConfig.isDeployServers());
+		assertTrue(cmaConfig.isDeployUsers());
+
+		p.setProperty("mlDeployWithCma", "false");
+		cmaConfig = new DefaultAppConfigFactory(new SimplePropertySource(p)).newAppConfig().getCmaConfig();
 		assertFalse(cmaConfig.isCombineRequests());
 		assertFalse(cmaConfig.isDeployAmps());
 		assertFalse(cmaConfig.isDeployDatabases());
 		assertFalse(cmaConfig.isDeployForests());
 		assertFalse(cmaConfig.isDeployPrivileges());
+		assertFalse(cmaConfig.isDeployProtectedPaths());
+		assertFalse(cmaConfig.isDeployQueryRolesets());
 		assertFalse(cmaConfig.isDeployRoles());
 		assertFalse(cmaConfig.isDeployServers());
 		assertFalse(cmaConfig.isDeployUsers());
-
-		p.setProperty("mlDeployWithCma", "true");
-		cmaConfig = new DefaultAppConfigFactory(new SimplePropertySource(p)).newAppConfig().getCmaConfig();
-		assertTrue(cmaConfig.isCombineRequests());
-		assertTrue(cmaConfig.isDeployAmps());
-		assertTrue(cmaConfig.isDeployDatabases());
-		assertTrue(cmaConfig.isDeployForests());
-		assertTrue(cmaConfig.isDeployPrivileges());
-		assertTrue(cmaConfig.isDeployProtectedPaths());
-		assertTrue(cmaConfig.isDeployQueryRolesets());
-		assertTrue(cmaConfig.isDeployRoles());
-		assertTrue(cmaConfig.isDeployServers());
-		assertTrue(cmaConfig.isDeployUsers());
 	}
 
 	@Test

src/test/java/com/marklogic/appdeployer/command/security/IncrementallyDeployUsersTest.java

@@ -1,5 +1,6 @@
 package com.marklogic.appdeployer.command.security;
 
+import com.marklogic.appdeployer.CmaConfig;
 import com.marklogic.appdeployer.command.AbstractIncrementalDeployTest;
 import com.marklogic.appdeployer.command.ResourceFileManagerImpl;
 import com.marklogic.mgmt.resource.security.RoleManager;
@@ -24,6 +25,9 @@ public void setup() {
 
 		// Have to turn resource merging off for this to work
 		appConfig.setMergeResources(false);
+
+		// And turn off CMA
+		appConfig.setCmaConfig(new CmaConfig());
 	}
 
 	@After
@@ -53,7 +57,7 @@ public void test() {
 	}
 
 	@Test
-	public void usersAndRoles() throws IOException  {
+	public void usersAndRoles() throws IOException {
 		this.originalManageClient = this.manageClient;
 
 		assertUsersDontExist();

src/test/java/com/marklogic/appdeployer/command/servers/DontUndeployDefaultServersTest.java

@@ -14,6 +14,9 @@ public class DontUndeployDefaultServersTest extends AbstractAppDeployerTest {
 
 	@Test
 	public void test() {
+		// CMA doesn't like the minimal Manage server file
+		appConfig.getCmaConfig().setDeployServers(false);
+
 		appConfig.setConfigDir(new ConfigDir(new File("src/test/resources/sample-app/default-servers")));
 
 		ServerManager mgr = new ServerManager(manageClient);