#58: added --configure-zookeeper-clusters

packaging/org.linkedin.glu.packaging-setup/src/cmdline/resources/conf/log4j.xml

@@ -25,6 +25,10 @@
     </layout>
   </appender>
 
+  <logger name="org.apache.zookeeper">
+    <level value="warn"/>
+  </logger>
+
   <root>
     <priority value="info"/>
     <appender-ref ref="ConsoleAppender"/>

packaging/org.pongasoft.glu.packaging-setup-impl/build.gradle

@@ -20,6 +20,7 @@ apply plugin: 'org.linkedin.release'
 dependencies {
   compile project(':utils:org.linkedin.glu.utils')
   compile project(':provisioner:org.linkedin.glu.provisioner-core')
+  compile spec.external.utilsZookeeperCliImpl
   compile spec.external.groovy
 
   runtime spec.external.commonsCli

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/AgentServerPackager.groovy

@@ -114,8 +114,11 @@ public class AgentServerPackager extends BasePackager
       tokens[CONFIG_TOKENS_KEY].GLU_AGENT_ZOOKEEPER_ROOT = metaModel.gluMetaModel.zooKeeperRoot
 
     Resource packagePath = outputFolder.createRelative(parts.join('-'))
-    copyInputPackage(packagePath)
-    configure(packagePath, tokens)
+    if(!dryMode)
+    {
+      copyInputPackage(packagePath)
+      configure(packagePath, tokens)
+    }
     return new PackagedArtifact(location: packagePath,
                                 host: metaModel.host.resolveHostAddress(),
                                 port: agentPort)

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/BasePackager.groovy

@@ -23,6 +23,8 @@ public class BasePackager
   Resource inputPackage
   Resource configsRoot
 
+  boolean dryMode = false
+
   void ensureVersion(String version)
   {
     if(!inputPackage.filename.endsWith("-${version}"))

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/ConsoleServerPackager.groovy

@@ -77,8 +77,11 @@ public class ConsoleServerPackager extends BasePackager
       parts << metaModel.name
 
     Resource packagePath =outputFolder.createRelative(parts.join('-'))
-    copyInputPackage(packagePath)
-    configure(packagePath, tokens)
+    if(!dryMode)
+    {
+      copyInputPackage(packagePath)
+      configure(packagePath, tokens)
+    }
     return new PackagedArtifact(location: packagePath,
                                 host: metaModel.host.resolveHostAddress(),
                                 port: metaModel.mainPort)

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/GluPackager.groovy

@@ -36,27 +36,49 @@ public class GluPackager
   Resource outputFolder
   Resource keysRoot
 
+  def packagedArtifacts = [:]
+
+  boolean dryMode = false
+
   void packageAll()
   {
     packageAgents()
+    packageConsoles()
+    packageZooKeeperClusters()
   }
 
   void packageAgents()
   {
-    gluMetaModel.agents.each {
-      PackagedArtifact pa = packageAgent(it)
-      println "Generated agent ${pa.location} ${pa.host}:${pa.port}"
+    gluMetaModel.agents.each { AgentMetaModel model ->
+      PackagedArtifact pa = packageAgent(model)
+      packagedArtifacts[model] = pa
+      if(!dryMode)
+        println "Generated agent ${pa.location} ${pa.host}:${pa.port}"
     }
-    gluMetaModel.consoles.values().each {
-      PackagedArtifact pa = packageConsole(it)
-      println "Generated console ${pa.location} ${pa.host}:${pa.port}"
+  }
+
+  void packageConsoles()
+  {
+    gluMetaModel.consoles.values().each { ConsoleMetaModel model ->
+      PackagedArtifact pa = packageConsole(model)
+      packagedArtifacts[model] = pa
+      if(!dryMode)
+        println "Generated console ${pa.location} ${pa.host}:${pa.port}"
     }
-    gluMetaModel.zooKeeperClusters.values().each {
-      def pas = packageZooKeeperCluster(it)
-      pas.zooKeepers.each { zki ->
-        println "Generated ZooKeeper instance ${zki.location} ${zki.host}:${zki.port}"
+  }
+
+  void packageZooKeeperClusters()
+  {
+    gluMetaModel.zooKeeperClusters.values().each { ZooKeeperClusterMetaModel model ->
+      def pas = packageZooKeeperCluster(model)
+      packagedArtifacts[model] = pas
+      if(!dryMode)
+      {
+        pas.zooKeepers.each { zki ->
+          println "Generated ZooKeeper instance ${zki.location} ${zki.host}:${zki.port}"
+        }
+        println "Generated ZooKeeper cluster ${pas.zooKeeperCluster.location} ${model.zooKeeperConnectionString}"
       }
-      println "Generated ZooKeeper cluster ${pas.zooKeeperCluster.location} ${it.zooKeeperConnectionString}"
     }
   }
 
@@ -69,7 +91,8 @@ public class GluPackager
                               inputPackage: getInputPackage('org.linkedin.glu.agent-server',
                                                             agentMetaModel.version),
                               configsRoot: configsRoot,
-                              metaModel: agentMetaModel)
+                              metaModel: agentMetaModel,
+                              dryMode: dryMode)
     packager.createPackage()
   }
 
@@ -82,7 +105,8 @@ public class GluPackager
                                 inputPackage: getInputPackage('org.linkedin.glu.console-server',
                                                               consoleMetaModel.version),
                                 configsRoot: configsRoot,
-                                metaModel: consoleMetaModel)
+                                metaModel: consoleMetaModel,
+                                dryMode: dryMode)
     packager.createPackage()
   }
 
@@ -95,7 +119,8 @@ public class GluPackager
                                    inputPackage: getInputPackage('org.linkedin.zookeeper-server',
                                                                  zooKeeperClusterMetaModel.zooKeepers[0].version),
                                    configsRoot: configsRoot,
-                                   metaModel: zooKeeperClusterMetaModel)
+                                   metaModel: zooKeeperClusterMetaModel,
+                                   dryMode: dryMode)
     packager.createPackage()
   }
 

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/SetupMain.groovy

@@ -19,12 +19,20 @@ import org.linkedin.glu.groovy.utils.shell.Shell
 import org.linkedin.glu.groovy.utils.shell.ShellImpl
 import org.linkedin.groovy.util.config.Config
 import org.linkedin.groovy.util.config.MissingConfigParameterException
+import org.linkedin.groovy.util.io.GroovyIOUtils
 import org.linkedin.groovy.util.log.JulToSLF4jBridge
+import org.linkedin.util.clock.Timespan
 import org.linkedin.util.io.resource.FileResource
 import org.linkedin.util.io.resource.Resource
 import org.linkedin.util.io.resource.ResourceChain
+import org.linkedin.zookeeper.cli.commands.UploadCommand
+import org.linkedin.zookeeper.client.ZKClient
+import org.pongasoft.glu.provisioner.core.metamodel.GluMetaModel
+import org.pongasoft.glu.provisioner.core.metamodel.ZooKeeperClusterMetaModel
 import org.pongasoft.glu.provisioner.core.metamodel.impl.builder.GluMetaModelBuilder
 
+import java.util.concurrent.TimeoutException
+
 /**
  * @author yan@pongasoft.com  */
 public class SetupMain
@@ -54,16 +62,20 @@ public class SetupMain
   protected def init(args)
   {
     cli = new CliBuilder(usage: './bin/setup.sh [-h]')
-    cli.d(longOpt: 'gen-dist', 'generate the distribution', args: 0, required: false)
+    cli.d(longOpt: 'gen-dist', 'generate the distributions', args: 0, required: false)
     cli.k(longOpt: 'gen-keys', 'generate the keys', args: 0, required: false)
+    cli.z(longOpt: 'configure-zookeeper-clusters', 'configure all zookeeper clusters', args: 0, required: false)
     cli.m(longOpt: 'meta-model', 'location of the meta model (multiple allowed)', args: 1, required: false)
     cli._(longOpt: 'configs-root', "location of the configs (multiple allowed) [default: ${defaultConfigsResource}]", args: 1, required: false)
     cli._(longOpt: 'packages-root', "location of the packages [default: ${defaultPackagesRootResource}]", args: 1, required: false)
     cli._(longOpt: 'keys-root', "location of the keys (if relative) [default: <outputFolder>/keys]", args: 1, required: false)
     cli._(longOpt: 'glu-root', "location of glu distribution [default: ${gluRootResource}]", args: 1, required: false)
-    cli.o(longOpt: 'outputFolder', 'output folder', args: 1, required: false)
+    cli._(longOpt: 'agents-only', "generate distribution for agents only", args: 0, required: false)
+    cli._(longOpt: 'consoles-only', "generate distribution for consoles only", args: 0, required: false)
+    cli._(longOpt: 'zookeeper-clusters-only', "generate distribution for ZooKeeper clusters only", args: 0, required: false)
+    cli.o(longOpt: 'output-folder', 'output folder', args: 1, required: false)
     cli._(longOpt: 'quiet', 'do not ask any question (use defaults)', args: 0, required: false)
-    cli.f(longOpt: 'setupConfigFile', 'the setup config file', args: 1, required: false)
+    cli.f(longOpt: 'setup-config-file', 'the setup config file', args: 1, required: false)
     cli.h(longOpt: 'help', 'display help')
 
     def options = cli.parse(args)
@@ -135,7 +147,7 @@ public class SetupMain
   {
     quiet = config.containsKey('quiet')
 
-    String out = Config.getOptionalString(config, 'outputFolder', null)
+    String out = Config.getOptionalString(config, 'output-folder', null)
 
     if(!out)
     {
@@ -146,7 +158,7 @@ public class SetupMain
     outputFolder = FileResource.create(new File(out))
 
     def actions = [
-      'gen-keys', 'gen-dist'].findAll {
+      'gen-keys', 'gen-dist', 'configure-zookeeper-clusters'].findAll {
       Config.getOptionalString(config, it, null)
     }
 
@@ -158,6 +170,9 @@ public class SetupMain
 
   }
 
+  /**
+   * --gen-keys command
+   */
   def gen_keys = {
     println "Generating keys..."
     char[] masterPassword = System.console().readPassword("Enter a master password:")
@@ -191,10 +206,88 @@ public class SetupMain
     }
   }
 
+  /**
+   * --gen-dist command
+   */
   def gen_dist = {
     println "Generating distributions"
 
-    // meta model
+    def packager = buildPackager(false)
+
+    if(Config.getOptionalBoolean(config, 'agents-only', false) ||
+       Config.getOptionalBoolean(config, 'consoles-only', false) ||
+       Config.getOptionalBoolean(config, 'zookeeper-clusters-only', false))
+    {
+      if(Config.getOptionalBoolean(config, 'agents-only', false))
+        packager.packageAgents()
+      if(Config.getOptionalBoolean(config, 'consoles-only', false))
+        packager.packageConsoles()
+      if(Config.getOptionalBoolean(config, 'zookeeper-clusters-only', false))
+        packager.packageZooKeeperClusters()
+    }
+    else
+    {
+      packager.packageAll()
+    }
+  }
+
+  /**
+   * --configure-zookeeper-clusters command
+   */
+  def configure_zookeeper_clusters = {
+    println "Configuring ZooKeeper clusters"
+
+    def packager = buildPackager(true)
+
+    packager.packageZooKeeperClusters()
+
+    def artifacts =
+      packager.packagedArtifacts.findAll { k, v -> k instanceof ZooKeeperClusterMetaModel}
+
+    artifacts.each { ZooKeeperClusterMetaModel model, def pas ->
+      configureZooKeeperCluster(model, pas.zooKeeperCluster.location)
+    }
+
+  }
+
+  protected void configureZooKeeperCluster(ZooKeeperClusterMetaModel model,
+                                           Resource location)
+  {
+    println "Configuring ZooKeeper cluster [${model.name}]"
+
+    def zkClient = new ZKClient(model.zooKeeperConnectionString,
+                                Timespan.parse("5s"),
+                                null)
+
+    zkClient.start()
+
+    try
+    {
+      zkClient.waitForStart(Timespan.parse('10s'))
+
+      GroovyIOUtils.eachChildRecurse(location.createRelative('conf').chroot('.')) { Resource child ->
+        if(!child.isDirectory())
+        {
+          println "uploading ${child.path} to ${model.zooKeeperConnectionString}"
+          UploadCommand cmd = new UploadCommand()
+          if(cmd.execute(zkClient, ['-f', child.file.canonicalPath, child.path]) != 0)
+            throw new AbortException("Error while uploading to ZooKeeper cluster [${model.zooKeeperConnectionString}]", 3)
+        }
+      }
+    }
+    catch(TimeoutException ignored)
+    {
+      throw new AbortException("could not connect to ZooKeeper [${model.zooKeeperConnectionString}]", 4)
+    }
+    finally
+    {
+      zkClient.destroy()
+    }
+
+  }
+
+  protected GluMetaModel loadGluMetaModel()
+  {
     def metaModels = config.'meta-models'
     if(!metaModels)
       throw new AbortException("--meta-model <arg> required", 2)
@@ -204,6 +297,14 @@ public class SetupMain
       builder.deserializeFromJsonResource(FileResource.create(metaModel))
     }
 
+    builder.toGluMetaModel()
+  }
+
+  protected GluPackager buildPackager(boolean dryMode)
+  {
+    // meta model
+    GluMetaModel gluMetaModel = loadGluMetaModel()
+
     // configsRoots
     def configsRoots = config.'configs-roots' ?: ['<default>']
     configsRoots = configsRoots.collect { String configRoot ->
@@ -224,17 +325,16 @@ public class SetupMain
                                             'keys-root',
                                             outputFolder.createRelative('keys').file.canonicalPath)
 
-    def packager = new GluPackager(shell: shell,
-                                   configsRoot: configsRoots,
-                                   packagesRoot: FileResource.create(packagesRoot),
-                                   outputFolder: outputFolder,
-                                   keysRoot: FileResource.create(keysRoot),
-                                   gluMetaModel: builder.toGluMetaModel())
-
-    packager.packageAll()
-
+    new GluPackager(shell: shell,
+                    configsRoot: configsRoots,
+                    packagesRoot: FileResource.create(packagesRoot),
+                    outputFolder: outputFolder,
+                    keysRoot: FileResource.create(keysRoot),
+                    gluMetaModel: gluMetaModel,
+                    dryMode: dryMode)
   }
 
+
   public static void main(String[] args)
   {
     SetupMain clientMain = new SetupMain()