#58: tweaked meta model

console/org.linkedin.glu.console-server/src/cmdline/resources/bin/consolectl.sh

@@ -67,6 +67,6 @@ fi
 # Java Tuning Options (heap & generations sizes; GC tuning)
 JVM_TUNING_OPTIONS="$JVM_SIZE $JVM_SIZE_NEW $JVM_SIZE_PERM $JVM_GC_TYPE $JVM_GC_OPTS $JVM_GC_LOG"
 
-JAVA_OPTIONS="$JAVA_OPTIONS $JVM_TUNING_OPTIONS -Dorg.linkedin.glu.console.config.location=$BASEDIR/conf/glu-console-webapp.groovy -Dorg.linkedin.glu.console.keys.dir=$BASEDIR/keys -Dorg.linkedin.glu.console.plugins.classpath=$PLUGINS_CLASSPATH -Dorg.linkedin.glu.console.root=$BASEDIR"
+JAVA_OPTIONS="$JAVA_OPTIONS $JVM_TUNING_OPTIONS $JVM_APP_INFO -Dorg.linkedin.glu.console.config.location=$BASEDIR/conf/glu-console-webapp.groovy -Dorg.linkedin.glu.console.keys.dir=$BASEDIR/keys -Dorg.linkedin.glu.console.plugins.classpath=$PLUGINS_CLASSPATH -Dorg.linkedin.glu.console.root=$BASEDIR"
 
 JAVA="$JAVA_CMD" JAVA_OPTIONS="$JAVA_OPTIONS" $JETTY_CMD "$@"

packaging/org.linkedin.glu.packaging-all/src/cmdline/resources/models/sample-production/staging-glu-meta-model.json.groovy

@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2013 Yan Pujante
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * This is a json/groovy DSL: you can use the power of groovy to 'dynamically' generate the output,
+ * for example using if/then conditions, looping for repeated values etc...
+ */
+
+/**
+ * Defining global versions
+ */
+metaModelVersion = '1.0.0' // the version of the format of this file
+gluVersion = '@glu.version@' // the glu version
+def zooKeeperVersion = '@zookeeper.version@' // the version for ZooKeeper distribution
+
+/**
+ * Step 1 (generate keys) produce a block of code that needs to be copied right below
+ * (note that if you do not use keys, simply replace the following section by:
+ *
+ * def keys = null
+ *
+ */
+////////////////////////////////////////
+def keys = [:]
+////////////////////////////////////////
+
+/**
+ * Define your fabrics here
+ */
+def stagingAlphaFabric = "staging-alpha"
+def stagingBetaFabric = "staging-beta"
+
+[stagingAlphaFabric, stagingBetaFabric].each { fabric ->
+  fabrics[fabric] = [
+    keys: keys,
+    console: 'stgConsole',
+    zooKeeperCluster: 'stgZkCluster'
+  ]
+}
+
+/**
+ * Define where each components should be installed on their target host.
+ * Note that a trailing / indicates it will be considered a directory in which to install
+ * the package.
+ *
+ * Otherwise provide the actual name you want (which you need to define for each type):
+ * '/opt/glu/agent-server'
+ */
+def installPath = '/opt/glu/'
+
+def installCommand = 'scp '
+
+/**
+ * Define your ZooKeeperClusters here.
+ * In general you need one ZooKeeper cluster per data center.
+ * In general a ZooKeeper cluster is 3 or 5 ZooKeeper instances.
+ */
+zooKeeperClusters << [
+  name: 'stgZkCluster',
+  zooKeepers: ['zk-host1', 'zk-host2', 'zk-host3'].collect { zkHost ->
+    [
+      version: zooKeeperVersion,
+      host: zkHost,
+      install: [
+        path: installPath,
+      ],
+      configTokens: [:] // map of config tokens if necessary
+    ]
+  }
+]
+
+/**
+ * Define your consoles here.
+ * You may need more than 1 console if you have multiple data centers (each console should be
+ * hosted in the same data center as the ZooKeeper cluster it is talking to).
+ *
+ * This section uses a more "real" database than the tutorial (MySql). You still need to install/
+ * configure the database outside of glu (creating the user needed below for example...)
+ *
+ */
+
+def consolePlugins = [
+  [
+    fqcn: 'org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'
+  ],
+  /*
+    add your own console plugins here
+     [  fqcn: 'com.acme.glu.MyPlugin',
+        classPath: ['http://xxx/glu-plugin.jar', ...]
+     ]
+   */
+]
+
+consoles << [
+  name: 'stgConsole',
+  host: 'console-host1',
+  install: [
+    path: installPath,
+  ],
+  plugins: consolePlugins,
+  dataSourceDriverUri: 'http://jcenter.bintray.com/mysql/mysql-connector-java/5.1.25/mysql-connector-java-5.1.25.jar',
+  configTokens: [
+    dataSource: """
+def dataSourceUrl = "jdbc:mysql://mysql-host1/glu"
+dataSource.dbCreate = "update"
+dataSource.url = dataSourceUrl
+dataSource.logSql=false // set to true for details (+ open trace logging level)
+dataSource.dialect = "org.hibernate.dialect.MySQL5InnoDBDialect"
+dataSource.driverClassName = "com.mysql.jdbc.Driver"
+dataSource.username= "xxx"
+dataSource.password = "yyy"
+""",
+  ]
+]
+
+/**
+ * Define your agents here...
+ */
+
+[
+  'agent-host1': stagingAlphaFabric,
+  'agent-host-2': stagingAlphaFabric,
+  'agent-host3': stagingBetaFabric].each { agentHost, fabric ->
+
+  agents << [
+    host: agentHost,
+    install: [
+      path: installPath,
+    ],
+    fabric: fabric,
+    configTokens: [:] // map of config tokens if necessary
+  ]
+}
+
+

packaging/org.linkedin.glu.packaging-all/src/cmdline/resources/models/tutorial/glu-meta-model.json.groovy

@@ -65,7 +65,11 @@ agents << [
 consoles << [
   name: 'tutorialConsole',
   host: 'localhost',
-  plugins: ['org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'],
+  plugins: [
+    [
+      fqcn: 'org.linkedin.glu.orchestration.engine.plugins.builtin.StreamFileContentPlugin'
+    ]
+  ],
   configTokens: [
     dataSource: """
 dataSource.dbCreate ='update'

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/GluPackager.groovy

@@ -27,11 +27,16 @@ import org.pongasoft.glu.provisioner.core.metamodel.AgentMetaModel
 import org.pongasoft.glu.provisioner.core.metamodel.ConsoleMetaModel
 import org.pongasoft.glu.provisioner.core.metamodel.GluMetaModel
 import org.pongasoft.glu.provisioner.core.metamodel.ZooKeeperClusterMetaModel
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 
 /**
  * @author yan@pongasoft.com  */
 public class GluPackager
 {
+  public static final String MODULE = GluPackager.class.getName();
+  public static final Logger log = LoggerFactory.getLogger(MODULE);
+
   public static final OneWayCodec SHA1 =
     OneWayMessageDigestCodec.createSHA1Instance('', HexaCodec.INSTANCE)
 
@@ -50,13 +55,15 @@ public class GluPackager
 
   boolean dryMode = false
 
-  void packageAll()
+  def packageAll()
   {
     packageAgents()
     packageConsoles()
     packageZooKeeperClusters()
     packageAgentCli()
     packageConsoleCli()
+
+    return packagedArtifacts
   }
 
   void packageAgents()
@@ -83,15 +90,15 @@ public class GluPackager
           // no need to generate the package!
           if(!dryMode)
           {
-            println "Skipped agent package ${pas.agentServer.location} ${pas.agentServer.host}:${pas.agentServer.port}"
+            log.info "Skipped agent package ${pas.agentServer.location} => ${pas.agentServer.host}:${pas.agentServer.port}"
           }
         }
       }
       else
       {
         packager.createPackage()
         if(!dryMode)
-          println "Generated agent package ${pas.agentServer.location} ${pas.agentServer.host}:${pas.agentServer.port}"
+          log.info "Generated agent package ${pas.agentServer.location} => ${pas.agentServer.host}:${pas.agentServer.port}"
         checksums[packageName] = checksum
       }
 
@@ -105,7 +112,7 @@ public class GluPackager
       PackagedArtifact pa = packageConsole(model)
       packagedArtifacts[model] = pa
       if(!dryMode)
-        println "Generated console package ${pa.location} ${pa.host}:${pa.port}"
+        log.info "Generated console package ${pa.location} => ${pa.host}:${pa.port}"
     }
   }
 
@@ -117,9 +124,9 @@ public class GluPackager
       if(!dryMode)
       {
         pas.zooKeepers.each { zki ->
-          println "Generated ZooKeeper instance ${zki.location} ${zki.host}:${zki.port}"
+          log.info "Generated ZooKeeper instance ${zki.location}  => ${zki.host}:${zki.port}"
         }
-        println "Generated ZooKeeper cluster ${pas.zooKeeperCluster.location} ${model.zooKeeperConnectionString}"
+        log.info "Generated ZooKeeper cluster ${pas.zooKeeperCluster.location} => ${model.zooKeeperConnectionString}"
       }
     }
   }

packaging/org.pongasoft.glu.packaging-setup-impl/src/main/groovy/org/pongasoft/glu/packaging/setup/KeysGenerator.groovy

@@ -17,6 +17,7 @@
 package org.pongasoft.glu.packaging.setup
 
 import org.linkedin.glu.groovy.utils.shell.Shell
+import org.linkedin.glu.groovy.utils.shell.ShellExecException
 import org.linkedin.util.codec.Base64Codec
 import org.linkedin.util.codec.Codec
 import org.linkedin.util.codec.CodecUtils
@@ -30,6 +31,10 @@ import org.pongasoft.glu.provisioner.core.metamodel.impl.KeysMetaModelImpl
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
+import java.security.Key
+import java.security.KeyStore
+import java.security.cert.Certificate
+
 /**
  * The purpose of this class is to generate keys and keystore for glu
  *
@@ -53,12 +58,7 @@ public class KeysGenerator
       keyalg: 'RSA',
       keysize: 2048,
       validity: 2000,
-      dname: [
-        cn: 'glu-cn',
-        ou: 'glu-ou',
-        o: 'glu-o',
-        c: 'glu-c'
-      ],
+      dname: "cn=glu-cn,ou=glu-ou,o=glu-o,c=glu-c"
     ]
 
   KeysMetaModelImpl keys
@@ -98,6 +98,8 @@ public class KeysGenerator
       keys.agentTrustStore = generateAgentTrustStore(keys.agentKeyStore)
       keys.consoleKeyStore = generateConsoleKeyStore()
       keys.consoleTrustStore = generateConsoleTrustStore(keys.consoleKeyStore)
+
+      generateConsolePem(keys.consoleKeyStore)
     }
 
     return keys
@@ -156,7 +158,7 @@ public class KeysGenerator
         '-keyalg', opts.keyalg,
         '-keysize', opts.keysize,
         '-validity', opts.validity,
-        '-dname', opts.dname.collect {k, v -> "${k}=${v}"}.join(', ')
+        '-dname', opts.dname
       ]
 
       def res = shell.exec(command: cmd)
@@ -254,7 +256,7 @@ public class KeysGenerator
         '-keyalg', opts.keyalg,
         '-keysize', opts.keysize,
         '-validity', opts.validity,
-        '-dname', opts.dname.collect {k, v -> "${k}=${v}"}.join(', ')
+        '-dname', opts.dname
       ]
 
       def res = shell.exec(command: cmd)
@@ -334,6 +336,74 @@ public class KeysGenerator
                                      storePassword: getEncryptedPasswords().consoleTrustStore)
   }
 
+  /**
+   * Step 5: generate a .pem file for use with curl
+   */
+  Resource generateConsolePem(KeyStoreMetaModel consoleKeyStore)
+  {
+    createResourceInOutputFolder('console.pem') { Resource consolePem ->
+
+      shell.withOutputStream(consolePem) { consolePemStream ->
+        KeyStore ks = KeyStore.getInstance("jks");
+
+        /* Load the key store. */
+        shell.withInputStream(toCanonicalPath(consoleKeyStore)) { InputStream stream ->
+          ks.load(stream, getPasswords().consoleKeyStore.toCharArray())
+        }
+
+        /* Save the private key. */
+        ByteArrayOutputStream derKey = new ByteArrayOutputStream()
+        derKey.withStream { kos ->
+          Key pvt = ks.getKey("console", getPasswords().consoleKey.toCharArray());
+          kos.write(pvt.getEncoded());
+        }
+
+        try
+        {
+          shell.exec(command: ['openssl', 'pkcs8', '-inform', 'der', '-nocrypt'],
+                     stdout: consolePemStream,
+                     stdin: new ByteArrayInputStream(derKey.toByteArray()),
+                     stderr: System.err,
+                     res: 'exitValue')
+        }
+        catch(ShellExecException ex)
+        {
+          log.warn("skipping generation of console.pem: could not execute openssl (is it installed?)")
+
+          if(log.isDebugEnabled())
+            log.debug("skipping generation of console.pem", ex)
+        }
+
+        /* Save the certificate. */
+        ByteArrayOutputStream derCert = new ByteArrayOutputStream()
+        derCert.withStream { cos ->
+          Certificate pub = ks.getCertificate("console");
+          cos.write(pub.getEncoded());
+        }
+
+        try
+        {
+          shell.exec(command: ['openssl', 'x509', '-inform', 'der'],
+                     stdout: consolePemStream,
+                     stdin: new ByteArrayInputStream(derCert.toByteArray()),
+                     stderr: System.err,
+                     res: 'exitValue')
+        }
+        catch(ShellExecException ex)
+        {
+          log.warn("skipping generation of console.pem: could not execute openssl (is it installed?)")
+
+          if(log.isDebugEnabled())
+            log.debug("skipping generation of console.pem", ex)
+        }
+
+      }
+
+
+      return consolePem
+    }
+  }
+
   private String toCanonicalPath(KeyStoreMetaModel keyStore)
   {
     if(keyStore.uri.isAbsolute())