#166: implemented commands configuration for console

commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemCommandExecutionIOStorage.groovy

@@ -183,10 +183,10 @@ public class FileSystemCommandExecutionIOStorage extends AbstractCommandExecutio
       if(exception != null)
       {
         if(exception instanceof Throwable)
-          args.exception = GluGroovyJsonUtils.exceptionToJSON(exception)
+          args.exception = GluGroovyJsonUtils.extractFullStackTrace(exception)
         else
         {
-          args.exception = exception.toString()
+          args.exception = GluGroovyJsonUtils.fromJSON(exception.toString())
           exception = GluGroovyJsonUtils.rebuildException(args.exception)
         }
       }

commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestFileSystemCommandExecutionIOStorage.groovy

@@ -23,7 +23,6 @@ import org.linkedin.groovy.util.io.fs.FileSystemImpl
 import org.linkedin.glu.commands.impl.GluCommandFactory
 import java.text.SimpleDateFormat
 import org.linkedin.groovy.util.json.JsonUtils
-import org.linkedin.groovy.util.collections.GroovyCollectionsUtils
 import org.linkedin.glu.commands.impl.CommandStreamStorage
 import org.linkedin.glu.commands.impl.StreamType
 import org.linkedin.util.clock.Timespan
@@ -33,6 +32,8 @@ import org.linkedin.glu.utils.io.MultiplexedInputStream
 import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils
 import org.linkedin.glu.groovy.utils.io.GluGroovyIOUtils
 import org.linkedin.glu.groovy.utils.plugins.PluginServiceImpl
+import org.linkedin.groovy.util.collections.IgnoreTypeComparator
+import org.linkedin.glu.groovy.utils.test.GluGroovyTestUtils
 
 /**
  * @author yan@pongasoft.com */
@@ -702,7 +703,7 @@ public class TestFileSystemCommandExecutionIOStorage extends GroovyTestCase
                              xtra0: 'x0',
                              startTime: startTime,
                              completionTime: completionTime,
-                             exception: GluGroovyJsonUtils.exceptionToJSON(exception),
+                             exception: GluGroovyJsonUtils.fromJSON(GluGroovyJsonUtils.exceptionToJSON(exception)),
                            ],
                            commandResource)
 
@@ -963,8 +964,7 @@ public class TestFileSystemCommandExecutionIOStorage extends GroovyTestCase
    */
   void assertEqualsIgnoreType(o1, o2)
   {
-    assertEquals(JsonUtils.prettyPrint(o1), JsonUtils.prettyPrint(o2))
-    assertTrue("expected <${o1}> but was <${o2}>", GroovyCollectionsUtils.compareIgnoreType(o1, o2))
+    GluGroovyTestUtils.assertEqualsIgnoreType(this, "expected <${o1}> but was <${o2}>", o1, o2)
   }
 
   /**

console/org.linkedin.glu.console-server/src/cmdline/resources/conf/glu-console-webapp.groovy

@@ -50,6 +50,19 @@ orchestration.engine.plugins = [
   'org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'
 ]
 
+// commands
+def commandsDir =
+  System.properties['org.linkedin.glu.console.commands.dir'] ?: "${System.properties['user.dir']}/commands"
+
+// storage type supported right now are 'filesystem' and 'memory'
+console.commandsService.storageType = 'filesystem'
+
+// when storage is filesystem => where the commands are stored
+console.commandsService.commandExecutionIOStorage.filesystem.rootDir = commandsDir
+
+// when storage is memory => how many elements maximum to store (then start evicting...)
+console.commandsService.commandExecutionIOStorage.memory.maxNumberOfElements = 25
+
 // The following property limits how many (leaf) steps get executed in parallel during a deployment
 // By default (undefined), it is unlimited
 // console.deploymentService.deployer.planExecutor.leafExecutorService.fixedThreadPoolSize = 100
@@ -201,4 +214,10 @@ console.defaults =
       [planType: "transition", displayName: "Stop", state: "stopped"],
     ],
 */
+    // features that can be turned on and off
+    features:
+    [
+      commands: true
+    ],
+
   ]

console/org.linkedin.glu.console-webapp/build.gradle

@@ -106,6 +106,7 @@ task lib(dependsOn: jar) << {
 console.keystorePath="${new File(keysDir, 'console.keystore').canonicalPath}"
 console.secretkeystorePath="${new File(secretKeyStoreDir, 'console.secretkeystore').canonicalPath}"
 console.truststorePath="${new File(keysDir, 'agent.truststore').canonicalPath}"
+console.commandsService.commandExecutionIOStorage.filesystem.rootDir="${buildDir}/commands"
 """
 
   // generate a hello world system for dev

console/org.linkedin.glu.console-webapp/grails-app/conf/BootStrap.groovy

@@ -137,7 +137,7 @@ class BootStrap {
       log.info "Successfully created (original) admin user. MAKE SURE YOU LOG IN AND CHANGE THE PASSWORD!"
     }
 
-    // initialzing default custom delta definition
+    // initializing default custom delta definition
     CustomDeltaDefinition defaultCustomDeltaDefinition =
       CustomDeltaDefinition.fromDashboard(consoleConfig.defaults.dashboard)
     if(defaultCustomDeltaDefinition)

console/org.linkedin.glu.console-webapp/grails-app/conf/Config.groovy

@@ -420,6 +420,9 @@ environments {
       'org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'
     ]
 
+    // storage type supported right now are 'filesystem' and 'memory'
+    console.commandsService.storageType = 'filesystem'
+
     // log4j configuration
     log4j = {
 
@@ -475,6 +478,9 @@ environments {
       'org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'
     ]
 
+    // storage type supported right now are 'filesystem' and 'memory'
+    console.commandsService.storageType = 'filesystem'
+
     // log4j configuration
     log4j = {
 

console/org.linkedin.glu.console-webapp/grails-app/conf/spring/resources.groovy

@@ -67,26 +67,27 @@ beans = {
     commandExecutionStorage(CommandExecutionStorageImpl)
 
     // IO Storage (command)
-    switch(Environment.current)
+    switch(consoleConfig.console.commandsService.storageType)
     {
-      case Environment.DEVELOPMENT:
-      case Environment.TEST:
-        commandExecutionIOStorage(MemoryCommandExecutionIOStorage) {
+      case 'filesystem':
+        def rootDir = new File(consoleConfig.console.commandsService.commandExecutionIOStorage.filesystem.rootDir)
+        commandExecutionFileSystem(FileSystemImpl, rootDir)
+
+        commandExecutionIOStorage(FileSystemCommandExecutionIOStorage) {
+          commandExecutionFileSystem = ref("commandExecutionFileSystem")
           pluginService = ref('pluginService')
         }
         break
 
-      default:
-        commandExecutionFileSystem(FileSystemImpl) { bean ->
-          bean.factoryMethod = "createTempFileSystem"
-          bean.destroyMethod = "destroy"
-        }
-
-        commandExecutionIOStorage(FileSystemCommandExecutionIOStorage) {
-          commandExecutionFileSystem = ref("commandExecutionFileSystem")
+      case 'memory':
+        commandExecutionIOStorage(MemoryCommandExecutionIOStorage) {
           pluginService = ref('pluginService')
+          maxNumberOfElements = consoleConfig.console.commandsService.commandExecutionIOStorage.memory.maxNumberOfElements ?: 25
         }
         break
+
+      default:
+        throw new IllegalArgumentException("unsupported storageType [${consoleConfig.console.commandsService.storageType}]")
     }
 
     commandsService(CommandsServiceImpl) {

console/org.linkedin.glu.console-webapp/grails-app/controllers/org/linkedin/glu/console/controllers/AgentsController.groovy

@@ -536,7 +536,6 @@ class AgentsController extends ControllerBase
     {
       response.sendError(HttpServletResponse.SC_NOT_FOUND,
                          'no such agent')
-      render ''
     }
   }
 

console/org.linkedin.glu.console-webapp/grails-app/controllers/org/linkedin/glu/console/controllers/CommandsController.groovy

@@ -101,7 +101,6 @@ public class CommandsController extends ControllerBase
     catch (NoSuchCommandExecutionException e)
     {
       response.sendError(HttpServletResponse.SC_NOT_FOUND)
-      render ''
     }
   }
 }

console/org.linkedin.glu.console-webapp/grails-app/controllers/org/linkedin/glu/console/controllers/PlanController.groovy

@@ -713,7 +713,6 @@ public class PlanController extends ControllerBase
     else
     {
       response.sendError HttpServletResponse.SC_NOT_FOUND
-      render ''
     }
   }
 

console/org.linkedin.glu.console-webapp/grails-app/utils/org/linkedin/glu/grails/utils/ConsoleConfig.groovy

@@ -58,7 +58,17 @@ class ConsoleConfig
   {
     return org.linkedin.groovy.util.config.Config.getOptionalBoolean(getDefault('features'),
                                                                      feature,
-                                                                     true)
+                                                                     false)
   }
 
+  void disableFeature(feature)
+  {
+    def features = defaults.features
+    if(!features)
+    {
+      features = [:]
+      defaults.features = features
+    }
+    features[feature] = false
+  }
 }

console/org.linkedin.glu.console-webapp/grails-app/views/commands/_command.gsp

@@ -15,5 +15,5 @@
   --}%
 
 <g:if test="${command}">
-  <div id="shell-${command.commandId}"><div class="shell span16"><g:link class="close" controller="agents" action="commands" id="${command.agent}">&times;</g:link><div class="cli"><span class="prompt">${command.username.encodeAsHTML()}@${command.agent.encodeAsHTML()}#</span>&nbsp;<span class="command">${command.command.encodeAsHTML()}</span> <g:if test="${command.redirectStderr}">2&gt;&amp;1 </g:if><span class="date">[<cl:formatDate time="${command.startTime}"/>]</span> <span class="exitValue">[${'$'}?=${command.exitValue?.encodeAsHTML()}]</span></div><g:if test="${!command.isExecuting}"><g:each in="['stdin', 'stderr', 'stdout']" var="streamType"><g:if test="${command.getTotalBytesCount(streamType) > 0}"><div id="${command.commandId}-${streamType}" class="${streamType}"><cl:renderCommandBytes command="${command}" streamType="${streamType}" onclick="renderCommandStream('${command.commandId}', '${streamType}', '${command.commandId}-${streamType}')"/></div></g:if></g:each><cl:renderJsonException exception="${command.exitError}"/><div class="cli"><span class="prompt">${command.username.encodeAsHTML()}@${command.agent.encodeAsHTML()}#</span>&nbsp;<span class="date">[<cl:formatDate time="${command.completionTime}"/>] (<cl:formatDuration duration="${command.duration}"/>)</span></div></g:if><g:else><img src="${resource(dir:'images',file:'spinner.gif')}" alt="Spinner" /> ... <g:link controller="agents" action="interruptCommand" id="${command.agent}" params="[commandId: command.commandId]">interrupt</g:link></g:else></div><g:if test="${command.isExecuting}"><g:javascript>setTimeout("renderCommand('${command.commandId}', 'shell-${command.commandId}')", '${params.refreshRate ?: 2000}')</g:javascript></g:if></div>
+  <div id="shell-${command.commandId}"><div class="shell span16"><g:link class="close" controller="agents" action="commands" id="${command.agent}">&times;</g:link><div class="cli"><span class="prompt">${command.username.encodeAsHTML()}@${command.agent.encodeAsHTML()}#</span>&nbsp;<span class="command">${command.command.encodeAsHTML()}</span> <g:if test="${command.redirectStderr}">2&gt;&amp;1 </g:if><span class="date">[<cl:formatDate time="${command.startTime}"/>]</span> <span class="exitValue">[${'$'}?=${command.exitValue?.encodeAsHTML()}]</span></div><g:if test="${!command.isExecuting}"><g:each in="['stdin', 'stderr', 'stdout']" var="streamType"><g:if test="${command.getTotalBytesCount(streamType) > 0}"><div id="${command.commandId}-${streamType}" class="${streamType}"><cl:renderCommandBytes command="${command}" streamType="${streamType}" onclick="renderCommandStream('${command.commandId}', '${streamType}', '${command.commandId}-${streamType}')"/></div></g:if></g:each><div class="cli"><span class="prompt">${command.username.encodeAsHTML()}@${command.agent.encodeAsHTML()}#</span>&nbsp;<span class="date">[<cl:formatDate time="${command.completionTime}"/>] (<cl:formatDuration duration="${command.duration}"/>)</span></div><cl:renderJsonException exception="${command.exitError}"/></g:if><g:else><img src="${resource(dir:'images',file:'spinner.gif')}" alt="Spinner" /> ... <g:link controller="agents" action="interruptCommand" id="${command.agent}" params="[commandId: command.commandId]">interrupt</g:link></g:else></div><g:if test="${command.isExecuting}"><g:javascript>setTimeout("renderCommand('${command.commandId}', 'shell-${command.commandId}')", '${params.refreshRate ?: 2000}')</g:javascript></g:if></div>
 </g:if>

orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandExecutionStream.groovy

@@ -55,10 +55,12 @@ public class CommandExecutionStream
           _stream = _limitedOutputStream
 
         streams[streamType.multiplexName] = _stream
+
+        c(this)
       }
     }
-
-    c(this)
+    else
+      c(this)
   }
 
   byte[] getBytes()

orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandsServiceImpl.groovy

@@ -183,6 +183,10 @@ public class CommandsServiceImpl implements CommandsService
       {
         // it is ok... we did not get any result during this amount of time
       }
+      catch(Throwable th)
+      {
+        log.warn("command execution generated an unknown exception [ignored]", th)
+      }
     }
 
     return command.id