Upgrade buttonweavers site to Ubuntu Xenial

.circleci/config.yml

@@ -1,22 +1,22 @@
 version: 2
 jobs:
   build:
-    machine:
-      image: circleci/classic:latest
+    docker:
+      - image: ubuntu:16.04
     steps:
       - checkout
       - run:
           name: Symlink /buttonmen
-          command: sudo ln -s /home/circleci/project /buttonmen
+          command: ln -s /root/project /buttonmen
       - run:
           name: Make sure log dir exists
           command: mkdir -p build/logs
       - run:
           name: Bootstrap puppet
-          command: sudo bash deploy/vagrant/bootstrap.sh
+          command: bash deploy/vagrant/bootstrap.sh
       - run:
           name: Run puppet
-          command: sudo puppet apply --modulepath=/buttonmen/deploy/vagrant/modules /buttonmen/deploy/circleci/manifests/init.pp
+          command: puppet apply --modulepath=/buttonmen/deploy/vagrant/modules /buttonmen/deploy/circleci/manifests/init.pp
       - run:
           name: Audit line termination of all files in the tree to match Button Men standard
           command: python deploy/circleci/audit_newlines.py
@@ -34,16 +34,20 @@ jobs:
           command: phpcs --report=checkstyle --report-file=./build/logs/checkstyle.xml --standard=./deploy/circleci/checkstyle/buttonmen.xml ./src/api ./src/engine
       - run:
           name: Run unit tests using PHPUnit and generates junit.xml and clover.xml
-          command: php /etc/php5/deploy-includes/phpunit.phar --bootstrap ./deploy/circleci/phpunit_bootstrap.php --log-junit build/logs/junit.xml --coverage-clover build/logs/clover.xml --debug test/
+          command: phpunit --bootstrap ./deploy/circleci/phpunit_bootstrap.php --log-junit build/logs/junit.xml --coverage-clover build/logs/clover.xml --debug test/
       - run:
           name: Install dummy API files created by PHPUnit into test webserver
-          command: sudo rsync -av src/api/dummy_data/ /var/www/api/dummy_data/
+          command: rsync -av src/api/dummy_data/ /var/www/api/dummy_data/
       - run:
           name: Verify dummy API files were fully installed
           command: bash ./deploy/circleci/verify_dummy_responder_files.sh src/api/dummy_data/ /var/www/api/dummy_data/
+      - run:
+          name: unpack phantomjs
+          command: /bin/tar xjf /usr/local/src/phantomjs-1.9.7-linux-x86_64.tar.bz2 --strip-components=1
+          working_directory: /usr/local
       - run:
           name: Run QUnit tests
-          command: /usr/bin/phantomjs --web-security=false /usr/local/etc/run-jscover-qunit.js http://localhost/test-ui/phantom-index.html
+          command: /usr/local/bin/phantomjs --web-security=false /usr/local/etc/run-jscover-qunit.js http://localhost/test-ui/phantom-index.html
       - run:
           name: Run python client unit tests
           command: /usr/bin/python ./test/tools/api-client/python/lib/test_bmapi.py
@@ -54,19 +58,17 @@ jobs:
           command: pdepend --jdepend-xml=./build/logs/jdepend.xml --jdepend-chart=./build/pdepend/dependencies.svg --overview-pyramid=build/pdepend/overview-pyramid.svg ./src
       - run:
           name: Generate pmd.xml using PHPMD
-          command: php /etc/php5/deploy-includes/phpmd.phar ./src xml ./deploy/circleci/pmd/buttonmen.xml --reportfile ./build/logs/pmd.xml
+          command: php /etc/php/7.0/deploy-includes/phpmd.phar ./src xml ./deploy/circleci/pmd/buttonmen.xml --reportfile ./build/logs/pmd.xml
       - run:
           name: Generate phploc.csv
-          command: php /etc/php5/deploy-includes/phploc.phar --log-csv ./build/logs/phploc.csv ./src
-      - run:
-          name: Generate API documentation using PHPDocumentor
-          command: phpdoc -d ./src -i 'api/DummyApiResponder.php' -t ./build/api
+          command: php /etc/php/7.0/deploy-includes/phploc.phar --log-csv ./build/logs/phploc.csv ./src
       - run:
           name: Aggregate tool output with PHP_CodeBrowser
-          command: php /etc/php5/deploy-includes/phpcb.phar --log ./build/logs --source ./src --output ./build/code-browser
+          command: php /etc/php/7.0/deploy-includes/phpcb.phar --log ./build/logs --source ./src --output ./build/code-browser
       - store_artifacts:
           path: build
 
 # TODO:
 # * get phpcpd working again
 # * get javascript code coverage testing working again using jscover or equivalent
+# * get phpdoc working again

deploy/circleci/manifests/init.pp

@@ -13,8 +13,6 @@
   include "apt::client"
   include "ntp::client"
   include "postfix::base"
-  include "user::buttonmen-devs"
-  include "sudo::buttonmen-devs"
 
   # Node configuration needed for the circleci buttonmen server
   include "apache::server::circleci"

deploy/vagrant/Vagrantfile

@@ -10,17 +10,17 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # please see the online documentation at vagrantup.com.
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "precise32"
+  config.vm.box = "ubuntu/xenial32"
 
   # The url from where the 'config.vm.box' box will be fetched if it
   # doesn't already exist on the user's system.
-  config.vm.box_url = "http://files.vagrantup.com/precise32.box"
+  #config.vm.box_url = "http://files.vagrantup.com/xenial32.box"
 
   # Create a forwarded port mapping which allows access to a specific port
   # within the machine from a port on the host machine. In the example below,
   # accessing "localhost:8080" will access port 80 on the guest machine.
   config.vm.network :forwarded_port, guest: 80, host: 8080
-  config.vm.hostname = "sandbox.buttonweavers.com"
+  #config.vm.hostname = "sandbox.buttonweavers.com"
 
   # Create a private network, which allows host-only access to the machine
   # using a specific IP.
@@ -64,9 +64,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     # firewall settings
     aws.security_groups = ["buttonmen-dev"]
 
-    #aws.ami = "ami-7747d01e"
     aws.instance_type = "t1.micro"
-    aws.ami = "ami-a53264cc"
+    aws.ami = "ami-0735ea082a1534cac"
 
     override.ssh.username = "ubuntu"
     override.ssh.private_key_path = "BUTTONMEN_SSH_PRIVATE_KEY_PATH"

deploy/vagrant/manifests/init.pp

@@ -30,7 +30,7 @@
   $puppet_timestamp = generate('/bin/date', '+%s')
 
   case "$operatingsystemrelease" {
-    "14.04": {
+    "14.04", "16.04": {
       $puppet_apache_sitesdir = "sites-enabled"
     }
     default: {
@@ -46,7 +46,7 @@
   include "sudo::buttonmen-devs"
 
   # Node configuration needed for the buttonmen server
-  include "apache::server"  
+  include "apache::server::vagrant"
   include "php::base"
   include "mysql::server"  
   include "buttonmen::server"

deploy/vagrant/modules/apache/manifests/init.pp

@@ -14,30 +14,34 @@
       require => Package["apache2"];
   }
 
-  # Customize apache default site
-  file {
-    "/etc/apache2/sites-available/default":
-      ensure => file,
-      content => template("apache/site_default.erb"),
-      notify => Service["apache2"];
-  }
-
   # Monitor the error log
   include "apache::server::feature::monitor-logs"
 }
 
-class apache::server::circleci {
+class apache::server::vagrant {
   # include the base class
   include "apache::server"
 
-  # Disable the default site, so the buttonmen site will load
   file {
+    # Customize apache default site
+    "/etc/apache2/sites-available/000-default.conf":
+      ensure => file,
+      content => template("apache/site_default.erb"),
+      notify => Service["apache2"];
+
+    # Enable the default site under the default name
     "/etc/apache2/sites-enabled/000-default.conf":
-      ensure => absent,
+      ensure => link,
+      target => "/etc/apache2/sites-available/000-default.conf",
       notify => Service["apache2"];
   }
 }
 
+class apache::server::circleci {
+  # same configuration as vagrant
+  include "apache::server::vagrant"
+}
+
 class apache::server::feature::monitor-logs {
 
   # Install the logtail package

deploy/vagrant/modules/apache/templates/site_default.erb

@@ -2,6 +2,26 @@
     ServerAdmin webmaster@localhost
 
     DocumentRoot /var/www
+
+    # Multiviews is what we use to redirect e.g. responder to responder.php
+    MultiviewsMatch Any
+
+    ## Set environment variables for buttonmen database access
+
+    # Database 1 (primary)
+    SetEnv DB1_HOST 127.0.0.1
+    SetEnv DB1_PORT 3306
+    SetEnv DB1_NAME buttonmen
+    SetEnv DB1_USER bmuser1
+    SetEnv DB1_PASS 79eWZGs2RohkIZMVElU6
+
+    # Database 2 (test)
+    SetEnv DB2_HOST 127.0.0.1
+    SetEnv DB2_PORT 3306
+    SetEnv DB2_NAME buttonmen_test
+    SetEnv DB2_USER bmtest
+    SetEnv DB2_PASS bmtestpass
+
     <Directory />
         Options FollowSymLinks
         AllowOverride None

deploy/vagrant/modules/apt/manifests/init.pp

@@ -3,4 +3,18 @@
     "apt_client_update":
       command => "/usr/bin/apt-get update";
   }
+
+  # Make sure generic base packages are installed, for the benefit
+  # of minimal environments like CircleCI docker
+  package {
+    "rsync": ensure => installed;
+    "wget": ensure => installed;
+  }
+
+  # Configure periodic apt cron job
+  file {
+    "/etc/apt/apt.conf.d/10periodic":
+      ensure => file,
+      content => template("apt/10periodic.erb");
+  }
 }

deploy/vagrant/modules/apt/templates/10periodic.erb

@@ -0,0 +1,9 @@
+##### /etc/apt/apt.conf.d/10periodic
+
+# Default xenial settings
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Download-Upgradeable-Packages "0";
+APT::Periodic::AutocleanInterval "0";
+
+# Do not automatically perform package updates
+APT::Periodic::Unattended-Upgrade "0";

deploy/vagrant/modules/buttonmen/manifests/init.pp

@@ -55,15 +55,15 @@
   exec {
     "buttonmen_src_rsync":
       command => "/usr/bin/rsync -a --delete /buttonmen/src/ /var/www/",
-      require => Package["apache2"];
+      require => [ Package["rsync"], Package["apache2"] ];
 
     "buttonmen_uitest_rsync":
       command => "/usr/bin/rsync -a --delete /buttonmen/test/src/ui/ /var/www/test-ui/",
       require => Exec["buttonmen_src_rsync"];
 
     "buttonmen_create_databases":
       command => "/usr/local/bin/create_buttonmen_databases",
-      require => [ Package["mysql-server"],
+      require => [ Service["mysql"],
                    Exec["buttonmen_src_rsync"] ];
 
 

deploy/vagrant/modules/buttonmen/templates/phpunit.php.erb

@@ -6,7 +6,7 @@
 
 // Include /etc/php5/deploy-includes for phpunit itself.
 // Include /buttonmen/src for now, so the individual unit tests can find the base classes
-set_include_path( "/etc/php5/deploy-includes:/buttonmen/src:" . ini_get( "include_path" ) );
+set_include_path( "/etc/php/7.0/deploy-includes:/buttonmen/src:" . ini_get( "include_path" ) );
 
 $BM_RAND_VALS = array();
 $BM_SKILL_RAND_VALS = array();

deploy/vagrant/modules/buttonmen/templates/run_buttonmen_tests.erb

@@ -30,7 +30,7 @@ echo "------------------------------------------------------------------------"
 # Run PHP unit tests
 echo "Running PHP unit tests"
 cd /buttonmen/src
-php /etc/php5/deploy-includes/phpunit.phar --bootstrap /usr/local/etc/buttonmen_phpunit.php /buttonmen/test
+phpunit --bootstrap /usr/local/etc/buttonmen_phpunit.php /buttonmen/test
 if [ "$?" = "0" ]; then
   echo "Passed"
 else

deploy/vagrant/modules/javascript/manifests/init.pp

@@ -1,12 +1,21 @@
 class javascript::type::circleci {
   package {
+    "npm": ensure => installed;
+    "nodejs-legacy": ensure => installed;
+
+    # we'll downgrade phantomjs later, but install it via apt to get its dependencies
     "phantomjs": ensure => installed;
-    "jscoverage": ensure => installed;
   }
 
   file {
     "/usr/local/etc/run-jscover-qunit.js":
       ensure => file,
       content => template("javascript/run-jscover-qunit.js.erb");
   }
+
+  exec {
+    "javascript_wget_phantomjs":
+      command => "/usr/bin/wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 -O /usr/local/src/phantomjs-1.9.7-linux-x86_64.tar.bz2",
+      creates => "/usr/local/src/phantomjs-1.9.7-linux-x86_64.tar.bz2";
+  }
 }

deploy/vagrant/modules/javascript/templates/install_phantomjs.erb

@@ -0,0 +1,13 @@
+#!/bin/bash
+##### Install phantomjs
+
+set -e
+set -x
+
+PHANTOMJS_VERSION_NAME="phantomjs-1.9.7-linux-x86_64"
+
+cd /tmp
+/usr/bin/wget https://bitbucket.org/ariya/phantomjs/downloads/${PHANTOMJS_VERSION_NAME}.tar.bz2 -O /tmp/${PHANTOMJS_VERSION_NAME}.tar.bz2
+sleep 10
+/bin/tar xjf ${PHANTOMJS_VERSION_NAME}.tar.bz2
+cp ${PHANTOMJS_VERSION_NAME}/bin/phantomjs /usr/local/bin/phantomjs

deploy/vagrant/modules/mysql/manifests/init.pp

@@ -4,9 +4,25 @@
   package {
     # Install mysql-server and php5-mysql packages
     "mysql-server": ensure => installed;
-    "php5-mysql": ensure => installed;
+    "php-mysql": ensure => installed;
 
     # Install python-mysqldb for use by helper scripts
     "python-mysqldb": ensure => installed;
   }
+
+  # Run mysqld
+  service {
+    "mysql":
+      ensure => running,
+      enable => true,
+      require => Package["mysql-server"];
+  }
+
+  # Customize mysqld for buttonmen use
+  file {
+    "/etc/mysql/mysql.conf.d/buttonmen.cnf":
+      ensure => file,
+      content => template("mysql/buttonmen.cnf.erb"),
+      notify => Service["mysql"];
+  }
 }

deploy/vagrant/modules/mysql/templates/buttonmen.cnf.erb

@@ -0,0 +1,11 @@
+# mysql configuration for buttonmen servers
+
+[mysqld]
+
+# Ubuntu 16.04 default sql_mode is:
+# ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
+# We need to remove:
+# * NO_ZERO_DATE - errors out on default 0 timestamps 
+# * STRICT_TRANS_TABLES - yields errors like: Field 'X' doesn't have a default value
+# * ONLY_FULL_GROUP_BY - a search in BMInterface::search_game_history violates this
+sql_mode=NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION