#58: generating agent, console, zk packages

packaging/org.linkedin.glu.packaging-all/build.gradle

@@ -50,13 +50,13 @@ cmdline {
     // keys
   resources << [
     from: agentKeys,
-    into: new File(assemblePackageFile, 'conf/tutorial/keys'),
+    into: new File(assemblePackageFile, 'models/tutorial/keys'),
     replaceTokens: false
   ]
 
   resources << [
     from: consoleKeys,
-    into: new File(assemblePackageFile, 'conf/tutorial/keys'),
+    into: new File(assemblePackageFile, 'models/tutorial/keys'),
     replaceTokens: false
   ]
 }

packaging/org.linkedin.glu.packaging-all/src/cmdline/resources/conf/tutorial/glu-meta-model.json.groovy → packaging/org.linkedin.glu.packaging-all/src/cmdline/resources/models/tutorial/glu-meta-model.json.groovy

@@ -19,7 +19,7 @@ metaModelVersion = '1.0.0'
 def fabric = 'glu-dev-1'
 
 def gluVersion = '@glu.version@'
-def zooKeeperVersion = '@zooKeeper.version@'
+def zooKeeperVersion = '@zookeeper.version@'
 
 def keys = [
   agentKeyStore: [

packaging/org.linkedin.glu.packaging-setup/src/cmdline/resources/configs/console-server/keys/agent.truststore.ctmpl

@@ -26,7 +26,7 @@ if(uri)
   else
   {
     def relativeKeyStoreFile = uri.toString()
-    def keyStore = packagerContext.keysRootDir.createRelative(relativeKeyStoreFile)
+    def keyStore = packagerContext.keysRoot.createRelative(relativeKeyStoreFile)
     packagerContext.shell.fetch(keyStore, toResource)
   }
 }

packaging/org.linkedin.glu.packaging-setup/src/cmdline/resources/configs/console-server/keys/console.keystore.ctmpl

@@ -26,7 +26,7 @@ if(uri)
   else
   {
     def relativeKeyStoreFile = uri.toString()
-    def keyStore = packagerContext.keysRootDir.createRelative(relativeKeyStoreFile)
+    def keyStore = packagerContext.keysRoot.createRelative(relativeKeyStoreFile)
     packagerContext.shell.fetch(keyStore, toResource)
   }
 }

packaging/org.linkedin.glu.packaging-setup/src/cmdline/resources/configs/zookeeper-cluster/fabrics/conf/@configTokens.zkRoot@/agents/fabrics/@configTokens.fabric@/config/agent.keystore.ctmpl

@@ -25,7 +25,7 @@ if(uri)
   else
   {
     def relativeKeyStoreFile = uri.toString()
-    def keyStore = packagerContext.keysRootDir.createRelative(relativeKeyStoreFile)
+    def keyStore = packagerContext.keysRoot.createRelative(relativeKeyStoreFile)
     packagerContext.shell.fetch(keyStore, toResource)
   }
 }

packaging/org.linkedin.glu.packaging-setup/src/cmdline/resources/configs/zookeeper-cluster/fabrics/conf/@configTokens.zkRoot@/agents/fabrics/@configTokens.fabric@/config/console.truststore.ctmpl

@@ -25,7 +25,7 @@ if(uri)
   else
   {
     def relativeKeyStoreFile = uri.toString()
-    def keyStore = packagerContext.keysRootDir.createRelative(relativeKeyStoreFile)
+    def keyStore = packagerContext.keysRoot.createRelative(relativeKeyStoreFile)
     packagerContext.shell.fetch(keyStore, toResource)
   }
 }

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/BasePackager.groovy

@@ -21,7 +21,7 @@ public class BasePackager
 
   Resource outputFolder
   Resource inputPackage
-  Resource configRoot
+  Resource configsRoot
 
   void ensureVersion(String version)
   {
@@ -68,7 +68,7 @@ public class BasePackager
 
   void processConfigs(String fromFolder, Map tokens, Resource toFolder)
   {
-    processConfigs(configRoot.createRelative(fromFolder), tokens, toFolder)
+    processConfigs(configsRoot.createRelative(fromFolder), tokens, toFolder)
   }
 
   void processConfigs(Resource fromFolder, Map tokens, Resource toFolder)

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/GluPackager.groovy

@@ -19,7 +19,9 @@ package org.pongasoft.glu.packaging.setup
 import org.linkedin.glu.groovy.utils.shell.Shell
 import org.linkedin.util.io.resource.Resource
 import org.pongasoft.glu.provisioner.core.metamodel.AgentMetaModel
+import org.pongasoft.glu.provisioner.core.metamodel.ConsoleMetaModel
 import org.pongasoft.glu.provisioner.core.metamodel.GluMetaModel
+import org.pongasoft.glu.provisioner.core.metamodel.ZooKeeperClusterMetaModel
 
 /**
  * @author yan@pongasoft.com  */
@@ -29,18 +31,91 @@ public class GluPackager
 
   Shell shell
 
-  Resource configsFolder
-  Resource distDir
+  Resource configsRoot
+  Resource packagesRoot
   Resource outputFolder
-  Resource keysRootDir
+  Resource keysRoot
+
+  void packageAll()
+  {
+    packageAgents()
+  }
 
   void packageAgents()
   {
-    gluMetaModel.agents.each { packageAgent(it) }
+    gluMetaModel.agents.each {
+      PackagedArtifact pa = packageAgent(it)
+      println "Generated agent ${pa.location} ${pa.host}:${pa.port}"
+    }
+    gluMetaModel.consoles.values().each {
+      PackagedArtifact pa = packageConsole(it)
+      println "Generated console ${pa.location} ${pa.host}:${pa.port}"
+    }
+    gluMetaModel.zooKeeperClusters.values().each {
+      def pas = packageZooKeeperCluster(it)
+      pas.zooKeepers.each { zki ->
+        println "Generated ZooKeeper instance ${zki.location} ${zki.host}:${zki.port}"
+      }
+      println "Generated ZooKeeper cluster ${pas.zooKeeperCluster.location} ${it.zooKeeperConnectionString}"
+    }
+  }
+
+  protected PackagedArtifact packageAgent(AgentMetaModel agentMetaModel)
+  {
+    def out = shell.mkdirs(outputFolder.createRelative('agents'))
+    def packager =
+      new AgentServerPackager(packagerContext: createPackagerContext(),
+                              outputFolder: out,
+                              inputPackage: getInputPackage('org.linkedin.glu.agent-server',
+                                                            agentMetaModel.version),
+                              configsRoot: configsRoot,
+                              metaModel: agentMetaModel)
+    packager.createPackage()
   }
 
-  private void packageAgent(AgentMetaModel agentMetaModel)
+  protected PackagedArtifact packageConsole(ConsoleMetaModel consoleMetaModel)
   {
-    shell.mkdirs(outputFolder.createRelative(agentMetaModel.fabric.name))
+    def out = shell.mkdirs(outputFolder.createRelative('consoles'))
+    def packager =
+      new ConsoleServerPackager(packagerContext: createPackagerContext(),
+                                outputFolder: out,
+                                inputPackage: getInputPackage('org.linkedin.glu.console-server',
+                                                              consoleMetaModel.version),
+                                configsRoot: configsRoot,
+                                metaModel: consoleMetaModel)
+    packager.createPackage()
   }
+
+  protected def packageZooKeeperCluster(ZooKeeperClusterMetaModel zooKeeperClusterMetaModel)
+  {
+    def out = shell.mkdirs(outputFolder.createRelative('zookeeper-clusters'))
+    def packager =
+      new ZooKeeperClusterPackager(packagerContext: createPackagerContext(),
+                                   outputFolder: out,
+                                   inputPackage: getInputPackage('org.linkedin.zookeeper-server',
+                                                                 zooKeeperClusterMetaModel.zooKeepers[0].version),
+                                   configsRoot: configsRoot,
+                                   metaModel: zooKeeperClusterMetaModel)
+    packager.createPackage()
+  }
+
+  protected PackagerContext createPackagerContext()
+  {
+    new PackagerContext(shell: shell,
+                        keysRoot: keysRoot ?: outputFolder.createRelative('keys'))
+  }
+
+  protected Resource getInputPackage(String name, String version)
+  {
+    def inputPackage = packagesRoot.createRelative("${name}-${version}")
+
+    if(!inputPackage.exists())
+      inputPackage = packagesRoot.createRelative("${name}-${version}.tgz")
+
+    if(!inputPackage.exists())
+      throw new FileNotFoundException("${inputPackage} does not exist")
+
+    return inputPackage
+  }
+
 }

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/PackagerContext.groovy

@@ -11,5 +11,5 @@ import org.linkedin.util.io.resource.Resource
 public class PackagerContext
 {
   Shell shell
-  Resource keysRootDir
+  Resource keysRoot
 }

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/SetupMain.groovy

@@ -22,19 +22,30 @@ import org.linkedin.groovy.util.config.MissingConfigParameterException
 import org.linkedin.groovy.util.log.JulToSLF4jBridge
 import org.linkedin.util.io.resource.FileResource
 import org.linkedin.util.io.resource.Resource
+import org.linkedin.util.io.resource.ResourceChain
+import org.pongasoft.glu.provisioner.core.metamodel.impl.builder.GluMetaModelBuilder
 
 /**
  * @author yan@pongasoft.com  */
 public class SetupMain
 {
+  public static class AbortException extends Exception
+  {
+    int exitValue = 0
+
+    AbortException(String message, int exitValue)
+    {
+      super(message)
+      this.exitValue = exitValue
+    }
+  }
+
   protected def config
   protected CliBuilder cli
   protected boolean quiet = false
   protected Resource outputFolder
   protected Shell shell = ShellImpl.createRootShell()
 
-  private int exitValue = 0
-
   SetupMain()
   {
     JulToSLF4jBridge.installBridge()
@@ -43,9 +54,16 @@ public class SetupMain
   protected def init(args)
   {
     cli = new CliBuilder(usage: './bin/setup.sh [-h]')
-    cli._(longOpt: 'gen-keys', 'generate the keys', args: 0, required: false)
-    cli._(longOpt: 'out', 'output directory', args: 1, required: false)
+    cli.d(longOpt: 'gen-dist', 'generate the distribution', args: 0, required: false)
+    cli.k(longOpt: 'gen-keys', 'generate the keys', args: 0, required: false)
+    cli.m(longOpt: 'meta-model', 'location of the meta model (multiple allowed)', args: 1, required: false)
+    cli._(longOpt: 'configs-root', "location of the configs (multiple allowed) [default: ${defaultConfigsResource}]", args: 1, required: false)
+    cli._(longOpt: 'packages-root', "location of the packages [default: ${defaultPackagesRootResource}]", args: 1, required: false)
+    cli._(longOpt: 'keys-root', "location of the keys (if relative) [default: <outputFolder>/keys]", args: 1, required: false)
+    cli._(longOpt: 'glu-root', "location of glu distribution [default: ${gluRootResource}]", args: 1, required: false)
+    cli.o(longOpt: 'outputFolder', 'output folder', args: 1, required: false)
     cli._(longOpt: 'quiet', 'do not ask any question (use defaults)', args: 0, required: false)
+    cli.f(longOpt: 'setupConfigFile', 'the setup config file', args: 1, required: false)
     cli.h(longOpt: 'help', 'display help')
 
     def options = cli.parse(args)
@@ -84,22 +102,51 @@ public class SetupMain
     }
   }
 
+  protected Resource getGluRootResource()
+  {
+    def gluRoot = Config.getOptionalString(config,
+                                           'glu-root',
+                                           userDirResource.createRelative('../..').file.canonicalPath)
+
+    FileResource.create(gluRoot)
+  }
+
+  protected String getUserDir()
+  {
+    System.getProperty('user.dir')
+  }
+
+  protected Resource getUserDirResource()
+  {
+    FileResource.create(userDir)
+  }
+
+  protected Resource getDefaultConfigsResource()
+  {
+    userDirResource.createRelative('configs')
+  }
+
+  protected Resource getDefaultPackagesRootResource()
+  {
+    gluRootResource.createRelative('packages')
+  }
+
   public void start()
   {
     quiet = config.containsKey('quiet')
 
-    String out = Config.getOptionalString(config, 'out', null)
+    String out = Config.getOptionalString(config, 'outputFolder', null)
 
     if(!out)
     {
       out = promptForValue("Enter the output directory",
-                           System.getProperty('user.pwd', System.getProperty('user.dir')))
+                           System.getProperty('user.pwd', userDir))
     }
 
     outputFolder = FileResource.create(new File(out))
 
     def actions = [
-      'gen-keys'].findAll {
+      'gen-keys', 'gen-dist'].findAll {
       Config.getOptionalString(config, it, null)
     }
 
@@ -139,11 +186,55 @@ public class SetupMain
     }
     catch(IllegalStateException e)
     {
-      exitValue = 1
-      println "${e.message} => if you want to generate new keys, either provide another folder or delete them first"
+      throw new AbortException("${e.message} => if you want to generate new keys, either provide another folder or delete them first",
+                               1)
     }
   }
 
+  def gen_dist = {
+    println "Generating distributions"
+
+    // meta model
+    def metaModels = config.'meta-models'
+    if(!metaModels)
+      throw new AbortException("--meta-model <arg> required", 2)
+
+    GluMetaModelBuilder builder = new GluMetaModelBuilder()
+    metaModels.each { String metaModel ->
+      builder.deserializeFromJsonResource(FileResource.create(metaModel))
+    }
+
+    // configsRoots
+    def configsRoots = config.'configs-roots' ?: ['<default>']
+    configsRoots = configsRoots.collect { String configRoot ->
+      if(configRoot == '<default>')
+        defaultConfigsResource
+      else
+        FileResource.create(configRoot)
+    }
+    configsRoots = ResourceChain.create(configsRoots)
+
+    // packagesRoot
+    def packagesRoot = Config.getOptionalString(config,
+                                                'packages-root',
+                                                defaultPackagesRootResource.file.canonicalPath)
+
+    // keysRoot
+    def keysRoot = Config.getOptionalString(config,
+                                            'keys-root',
+                                            outputFolder.createRelative('keys').file.canonicalPath)
+
+    def packager = new GluPackager(shell: shell,
+                                   configsRoot: configsRoots,
+                                   packagesRoot: FileResource.create(packagesRoot),
+                                   outputFolder: outputFolder,
+                                   keysRoot: FileResource.create(keysRoot),
+                                   gluMetaModel: builder.toGluMetaModel())
+
+    packager.packageAll()
+
+  }
+
   public static void main(String[] args)
   {
     SetupMain clientMain = new SetupMain()
@@ -160,9 +251,14 @@ public class SetupMain
         println e
         clientMain.cli.usage()
       }
+      catch(AbortException e)
+      {
+        System.err.println(e.message)
+        System.exit(e.exitValue)
+      }
     }
 
-    System.exit(clientMain.exitValue)
+    System.exit(0)
   }
 
   protected def getConfig(cli, options)
@@ -180,6 +276,10 @@ public class SetupMain
       if(options.hasOption(option.longOpt))
       {
         properties[option.longOpt] = options[option.longOpt]
+        def collectionOptionName = "${option.longOpt}s".toString()
+        def array = options."${collectionOptionName}"
+        if(array)
+          properties[collectionOptionName] = array
       }
     }
 

packaging/org.pongasoft.glu.packaging-setup-impl/src/test/groovy/test/setup/BasePackagerTest.groovy

@@ -133,7 +133,7 @@ public abstract class BasePackagerTest extends GroovyTestCase
   protected PackagerContext createPackagerContext(Shell shell)
   {
     new PackagerContext(shell: shell,
-                        keysRootDir: keysRootResource)
+                        keysRoot: keysRootResource)
   }
 
   /**