WIP: make it so wp initial build take the RDS host into account

util/mysqlpreparenoimport_wp.sh

@@ -9,6 +9,7 @@
 myFile='/etc/mysql/debian.cnf'
 myUser=`grep user $myFile | awk '{print $3}' | uniq`
 myPass=`grep password $myFile | awk '{print $3}' | uniq`
+myHost=`grep host $myFile | awk '{print $3}' | uniq`
 
 
 # Args that must be passed
@@ -47,22 +48,22 @@ fi
 ORIGDB=$NEWDB
 
 while [  $COUNTER -lt 99 ]; do
-  DB=`mysql -u$myUser -p$myPass -Bse 'show databases'| egrep "^${NEWDB}$"`
+  DB=`mysql -u$myUser -p$myPass -h$myHost -Bse 'show databases'| egrep "^${NEWDB}$"`
   if [ "$DB" = "$NEWDB" ]; then
     echo "The database $NEWDB already exists"
     let COUNTER=COUNTER+1
     NEWDB=${ORIGDB}_${COUNTER}
     continue
   else
     echo "Creating a database for $NEWDB"
-    mysqladmin -u$myUser -p$myPass create $NEWDB || exit 1
+    mysqladmin -u$myUser -p$myPass -h$myHost create $NEWDB || exit 1
     echo "Generating a user/pass for database $NEWDB"
-    mysql -u$myUser -p$myPass -e "GRANT ALL ON \`$NEWDB\`.* to '$NEWDB'@'localhost' IDENTIFIED BY '$PASS'" || exit 1
+    mysql -u$myUser -p$myPass -h$myHost -e "GRANT ALL ON \`$NEWDB\`.* to '$NEWDB'@'$myHost' IDENTIFIED BY '$PASS'" || exit 1
     break
   fi
 done
 
-DB=`mysql -u$myUser -p$myPass -Bse 'show databases'| egrep "^${NEWDB}$"`
+DB=`mysql -u$myUser -p$myPass -h$myHost -Bse 'show databases'| egrep "^${NEWDB}$"`
 # Only do any importing if the database exists ( we should have failed above
 if [ $? -eq 0 ]; then
   #########################################################################
@@ -83,7 +84,7 @@ if [ $? -eq 0 ]; then
   #
   #########################################################################
 
-  wp --path=$SITE_ROOT --allow-root core config --dbname=$NEWDB --dbuser=$NEWDB --dbpass=$PASS
+  wp --path=$SITE_ROOT --allow-root core config --dbname=$NEWDB --dbuser=$NEWDB --dbpass=$PASS --dbhost=$myHost
   wp --path=$SITE_ROOT --allow-root core install --url=$URL --title=$NEWDB --admin_user=codeenigma --admin_email=sysadm@codeenigma.com --admin_password=$PASS
 else
   echo "Something went wrong and we couldn't import a database or populate wp-config.php. Maybe we couldn't create the database?"

wordpress/InitialBuild.py

@@ -4,12 +4,13 @@
 import string
 # Custom Code Enigma modules
 import common.Utils
+import common.MySQL
 # Needed to get variables set in modules back into the main script
 from common.Utils import *
 
 # Stuff to do when this is the initial build
 @task
-def initial_build(repo, url, branch, build, profile, webserverport):
+def initial_build(repo, url, branch, build, buildtype, profile, webserver, webserverport, config, db_name, db_username, db_password, mysql_version, mysql_config, install_type, cluster=False, autoscale=False, rds=False):
   print "===> This looks like the first build! We have some things to do.."
 
   print "===> Setting the live document root symlink"
@@ -23,38 +24,46 @@ def initial_build(repo, url, branch, build, profile, webserverport):
   print "===> Preparing the database"
   # this process creates a database, database user/pass, imports the db dump from the repo
   # and generates the wp-config.php database file.
-  newpass = common.Utils._gen_passwd()
 
-  # Check if a db/ directory exists first.
-  db_dir = False
-  with settings(warn_only=True):
-    if run("find /var/www/%s_%s_%s -maxdepth 1 -type d -name db | egrep '.*'" % (repo, branch, build)).return_code == 0:
-      db_dir = True
+  # We can default these to None, mysql_new_database() will sort itself out
+  list_of_app_servers = None
+  db_host = None
+  # For clusters we need to do some extra things
+  if cluster or autoscale:
+    # This is the Database host that we need to insert into wordpress config. It is different from the main db host because it might be a floating IP
+    db_host = config.get('WPDBHost', 'dbhost')
+    print "db is %s" % (db_host)
+    # Convert a list of apps back into a string, to pass to the MySQL new database function for setting appropriate GRANTs to the database
+    list_of_app_servers = env.roledefs['app_all']
+    print "app list is %s" % (list_of_app_servers)
+  if cluster and config.has_section('AppIPs'):
+    list_of_app_servers = env.roledefs['app_ip_all']
 
-  if db_dir:
-    sudo("bunzip2 /var/www/%s_%s_%s/db/*.sql.bz2" % (repo, branch, build))
-    # Copying the database script to the server.
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-    if put(script_dir + '/../util/mysqlprepare_wp.sh', '/home/jenkins', mode=0755).failed:
-      raise SystemExit("Could not copy the database script to the application server, aborting because we won't be able to make a database")
-    else:
-      print "===> Database preparation script mysqlprepare_wp.sh copied to %s:/home/jenkins/mysqlprepare_wp.sh" % (env.host)
-    # Note, Wordpress version of script, mysqlprepare_wp.sh <databasename> <databasepass> <site_root> <branch> <dumpfile>
-    sudo("/home/jenkins/mysqlprepare_wp.sh %s %s /var/www/live.%s.%s %s $(find /var/www/%s_%s_%s/db -type f -name *.sql)" % (repo, newpass, repo, branch, branch, repo, branch, build))
-  else:
-    sitedir = "/var/www/%s_%s_%s/www" % (repo, branch, build)
-    # Copying the database script to the server.
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-    if put(script_dir + '/../util/mysqlpreparenoimport_wp.sh', '/home/jenkins', mode=0755).failed:
-      raise SystemExit("Could not copy the database script to the application server, aborting because we won't be able to make a database")
+  # Prepare the database
+  # We'll get back db_name, db_username, db_password and db_host from this call as a list in new_database
+  new_database = common.MySQL.mysql_new_database(repo, buildtype, rds, db_name, db_host, db_username, mysql_version, db_password, mysql_config, list_of_app_servers)
+
+  print "===> Waiting 10 seconds to let MySQL internals catch up"
+  time.sleep(10)
+
+  # Copy wp-config.php into place
+  with settings(warn_only=True):
+    if run("stat /var/www/live.%s.%s/wp-config.php.%s" % (repo, branch, branch)).return_code == 0:
+      sudo("cp /var/www/live.%s.%s/wp-config.php.%s /var/www/live.%s.%s/wp-config.php" % (repo, branch, branch, repo, branch))
     else:
-      print "===> Database preparation script mysqlpreparenoimport_wp.sh copied to %s:/home/jenkins/mysqlpreparenoimport_wp.sh" % (env.host)
-    sudo("/home/jenkins/mysqlpreparenoimport_wp.sh %s %s %s %s %s %s" % (repo, newpass, sitedir, branch, profile, url))
+      print "No wp-config.php.%s file, continuing..." % branch
 
-  webserver = "nginx"
+
+  # wp-cli site install.
+  if install_type == "www":
+    install_path = "/var/www/live.%s.%s/www" % (repo, branch)
+  else:
+    install_path = "/var/www/live.%s.%s" % (repo, branch)
+  sudo("wp --path=%s --allow-root core config --dbname=%s --dbuser=%s --dbpass=%s --dbhost=%s" % (install_path, new_database[0], new_database[1], new_database[2], new_database[3]))
+  sudo("wp --path=%s --allow-root core install --url=%s --title=%s --admin_user=codeenigma --admin_email=sysadm@codeenigma.com --admin_password=%s" % (install_path, url, new_database[0], new_database[2]))
 
   print "===> Setting up an %s vhost" % webserver
-  # Copy Nginx vhost to server(s)
+  # Copy vhost to server(s)
   print "===> Placing new copies of dummy vhosts for %s before proceeding" % webserver
   script_dir = os.path.dirname(os.path.realpath(__file__))
   if put(script_dir + '/../util/vhosts/%s/wp-*' % webserver, '/etc/%s/sites-available' % webserver, mode=0755, use_sudo=True).failed:

wordpress/fabfile.py

@@ -22,15 +22,23 @@
 # pesky 'stdin is not a tty' messages when using sudo
 env.shell = '/bin/bash -c'
 
-# Read the config.ini file from repo, if it exists
-config = common.ConfigFile.read_config_file()
+
 
 
 ######
 # New 'main()' task which should replace the deployment.sh wrapper, and support repo -> host mapping
 #####
 @task
-def main(repo, repourl, build, branch, buildtype, url=None, keepbuilds=20, profile="minimal", webserverport='8080', php_ini_file=None):
+def main(repo, repourl, build, branch, buildtype, url=None, keepbuilds=20, profile="minimal", webserver='nginx', webserverport='8080', php_ini_file=None, mysql_version=5.5, mysql_config='/etc/mysql/debian.cnf', config_filename='config.ini', config_fullpath=False, install_type='', cluster=False, autoscale=None, rds=False):
+
+  if config_fullpath == "False":
+    config_fullpath = False
+  if config_fullpath == "True":
+    config_fullpath = True
+
+  # Read the config.ini file from repo, if it exists
+  config = common.ConfigFile.buildtype_config_file(buildtype, config_filename, fullpath=config_fullpath)
+
   # We need to iterate through the options in the map and find the right host based on
   # whether the repo name matches any of the options, as they may not be exactly identical
   if config.has_section(buildtype):
@@ -42,6 +50,9 @@ def main(repo, repourl, build, branch, buildtype, url=None, keepbuilds=20, profi
            env.host = entry.strip()
            print "===> Host is %s" % env.host
            break
+  aws_credentials = common.ConfigFile.return_config_item(config, "AWS", "aws_credentials", "string", "/home/jenkins/.aws/credentials")
+  aws_autoscale_group = common.ConfigFile.return_config_item(config, "AWS", "aws_autoscale_group", "string", "prod-asg-prod")
+  common.Utils.define_roles(config, cluster, autoscale, aws_credentials, aws_autoscale_group)
 
   # Didn't find any host in the map for this project.
   if env.host is None:
@@ -56,9 +67,15 @@ def main(repo, repourl, build, branch, buildtype, url=None, keepbuilds=20, profi
   # Can be set in the config.ini [Build] section
   ssh_key = common.ConfigFile.return_config_item(config, "Build", "ssh_key")
   notifications_email = common.ConfigFile.return_config_item(config, "Build", "notifications_email")
+  php_ini_file = common.ConfigFile.return_config_item(config, "Build", "php_ini_file", "string", php_ini_file)
+  db_name = common.ConfigFile.return_config_item(config, "Database", "db_name")
+  db_username = common.ConfigFile.return_config_item(config, "Database", "db_username")
+  db_password = common.ConfigFile.return_config_item(config, "Database", "db_password")
   # Need to keep potentially passed in 'url' value as default
   url = common.ConfigFile.return_config_item(config, "Build", "url", "string", url)
-  php_ini_file = common.ConfigFile.return_config_item(config, "Build", "php_ini_file", "string", php_ini_file)
+
+  # Need to keep potentially passed in MySQL version and config path as defaults
+  mysql_version = common.ConfigFile.return_config_item(config, "Database", "mysql_version", "string", mysql_version)
 
   # Set a URL if one wasn't already provided
   if url is None:
@@ -86,7 +103,8 @@ def main(repo, repourl, build, branch, buildtype, url=None, keepbuilds=20, profi
     print "===> Looks like the site %s doesn't exist. We'll try and install it..." % url
     try:
       common.Utils.clone_repo(repo, repourl, branch, build, None, ssh_key)
-      InitialBuild.initial_build(repo, url, branch, build, profile, webserverport)
+      InitialBuild.initial_build(repo, url, branch, build, buildtype, profile, webserver, webserverport, config, db_name, db_username, db_password, mysql_version, mysql_config, install_type, cluster, autoscale, rds)
+
       # Unset CLI PHP version if we need to
       if php_ini_file:
         run("export PHPRC=''")