Db collation standardisation (librenms#5932)

* initial work to standardise collation

* more updates

* final bits for collation update (famous last words)

* Rename 165.sql to 166.sql

* Update schema version inside 166.sql

* moved schema file 166->171

Author: laf

build-base.php

@@ -23,6 +23,9 @@
 }
 
 $select = mysqli_select_db($database_link, $config['db_name']);
+mysqli_query($database_link, "SET NAMES 'utf8'");
+mysqli_query($database_link, "SET CHARACTER SET 'utf8'");
+mysqli_query($database_link, "SET COLLATION_CONNECTION = 'utf8_unicode_ci'");
 if ($select === false) {
     echo 'ERROR: Cannot select database: '.mysqli_error($database_link)."\n";
     exit(1);

build.sql

@@ -68,12 +68,9 @@ CREATE TABLE IF NOT EXISTS `frequency` ( `freq_id` int(11) NOT NULL auto_increme
 CREATE TABLE IF NOT EXISTS `current` ( `current_id` int(11) NOT NULL auto_increment, `device_id` int(11) NOT NULL default '0', `current_oid` varchar(64) NOT NULL, `current_index` varchar(8) NOT NULL, `current_type` varchar(32) NOT NULL, `current_descr` varchar(32) NOT NULL default '', `current_precision` int(11) NOT NULL default '1', `current_current` float default NULL, `current_limit` float default NULL, `current_limit_warn` float default NULL, `current_limit_low` float default NULL, PRIMARY KEY  (`current_id`), KEY `current_host` (`device_id`)) ENGINE=MyISAM AUTO_INCREMENT=189 DEFAULT CHARSET=latin1;
 CREATE TABLE IF NOT EXISTS `ucd_diskio` (  `diskio_id` int(11) NOT NULL AUTO_INCREMENT,  `device_id` int(11) NOT NULL,  `diskio_index` int(11) NOT NULL,  `diskio_descr` varchar(32) NOT NULL,  PRIMARY KEY (`diskio_id`)) ENGINE=MyISAM  DEFAULT CHARSET=latin1 AUTO_INCREMENT=11 ;
 CREATE TABLE IF NOT EXISTS `applications` (  `app_id` int(11) NOT NULL AUTO_INCREMENT,  `device_id` int(11) NOT NULL,  `app_type` varchar(64) NOT NULL,  PRIMARY KEY (`app_id`)) ENGINE=MyISAM  DEFAULT CHARSET=latin1 AUTO_INCREMENT=5 ;
-## 0.10.6
 CREATE TABLE IF NOT EXISTS `sensors` ( `sensor_id` int(11) NOT NULL auto_increment, `sensor_class` varchar(64) NOT NULL, `device_id` int(11) NOT NULL default '0', `sensor_oid` varchar(64) NOT NULL, `sensor_index` varchar(8) NOT NULL, `sensor_type` varchar(32) NOT NULL, `sensor_descr` varchar(32) NOT NULL default '', `sensor_precision` int(11) NOT NULL default '1', `sensor_current` float default NULL, `sensor_limit` float default NULL, `sensor_limit_warn` float default NULL, `sensor_limit_low` float default NULL, `sensor_limit_low_warn` float default NULL, PRIMARY KEY  (`sensor_id`), KEY `sensor_host` (`device_id`)) ENGINE=MyISAM AUTO_INCREMENT=189 DEFAULT CHARSET=latin1;
 CREATE TABLE IF NOT EXISTS `ports_adsl` (  `interface_id` int(11) NOT NULL,  `port_adsl_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,  `adslLineCoding` varchar(8) COLLATE utf8_bin NOT NULL,  `adslLineType` varchar(16) COLLATE utf8_bin NOT NULL,  `adslAtucInvVendorID` varchar(8) COLLATE utf8_bin NOT NULL,  `adslAtucInvVersionNumber` varchar(8) COLLATE utf8_bin NOT NULL,  `adslAtucCurrSnrMgn` decimal(5,1) NOT NULL,  `adslAtucCurrAtn` decimal(5,1) NOT NULL,  `adslAtucCurrOutputPwr` decimal(5,1) NOT NULL,  `adslAtucCurrAttainableRate` int(11) NOT NULL,  `adslAtucChanCurrTxRate` int(11) NOT NULL,  `adslAturInvSerialNumber` varchar(8) COLLATE utf8_bin NOT NULL,  `adslAturInvVendorID` varchar(8) COLLATE utf8_bin NOT NULL,  `adslAturInvVersionNumber` varchar(8) COLLATE utf8_bin NOT NULL,  `adslAturChanCurrTxRate` int(11) NOT NULL,  `adslAturCurrSnrMgn` decimal(5,1) NOT NULL,  `adslAturCurrAtn` decimal(5,1) NOT NULL,  `adslAturCurrOutputPwr` decimal(5,1) NOT NULL,  `adslAturCurrAttainableRate` int(11) NOT NULL,  UNIQUE KEY `interface_id` (`interface_id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-## 0.10.7
 CREATE TABLE IF NOT EXISTS `perf_times` (  `type` varchar(8) NOT NULL,  `doing` varchar(64) NOT NULL,  `start` int(11) NOT NULL,  `duration` double(5,2) NOT NULL,  `devices` int(11) NOT NULL,  KEY `type` (`type`)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-## 0.10.7.1
 CREATE TABLE IF NOT EXISTS `device_graphs` (  `device_id` int(11) NOT NULL,  `graph` varchar(32) COLLATE utf8_unicode_ci NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 CREATE TABLE IF NOT EXISTS `graph_types` (  `graph_type` varchar(32) COLLATE utf8_unicode_ci NOT NULL,  `graph_subtype` varchar(32) COLLATE utf8_unicode_ci NOT NULL,  `graph_section` varchar(32) COLLATE utf8_unicode_ci NOT NULL,  `graph_descr` varchar(64) COLLATE utf8_unicode_ci NOT NULL,  `graph_order` int(11) NOT NULL,  KEY `graph_type` (`graph_type`),  KEY `graph_subtype` (`graph_subtype`),  KEY `graph_section` (`graph_section`)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 INSERT INTO `graph_types` (`graph_type`, `graph_subtype`, `graph_section`, `graph_descr`, `graph_order`) VALUES('device', 'bits', 'netstats', 'Total Traffic', 0),('device', 'hr_users', 'system', 'Users Logged In', 0),('device', 'ucd_load', 'system', 'Load Averages', 0),('device', 'ucd_cpu', 'system', 'Detailed Processor Usage', 0),('device', 'ucd_memory', 'system', 'Detailed Memory Usage', 0),('device', 'netstat_tcp', 'netstats', 'TCP Statistics', 0),('device', 'netstat_icmp_info', 'netstats', 'ICMP Informational Statistics', 0),('device', 'netstat_icmp_stat', 'netstats', 'ICMP Statistics', 0),('device', 'netstat_ip', 'netstats', 'IP Statistics', 0),('device', 'netstat_ip_frag', 'netstats', 'IP Fragmentation Statistics', 0),('device', 'netstat_udp', 'netstats', 'UDP Statistics', 0),('device', 'netstat_snmp', 'netstats', 'SNMP Statistics', 0),('device', 'temperatures', 'system', 'Temperatures', 0),('device', 'mempools', 'system', 'Memory Pool Usage', 0),('device', 'processors', 'system', 'Processor Usage', 0),('device', 'storage', 'system', 'Filesystem Usage', 0),('device', 'hr_processes', 'system', 'Running Processes', 0),('device', 'uptime', 'system', 'System Uptime', 0),('device', 'ipsystemstats_ipv4', 'netstats', 'IPv4 Packet Statistics', 0),('device', 'ipsystemstats_ipv6_frag', 'netstats', 'IPv6 Fragmentation Statistics', 0),('device', 'ipsystemstats_ipv6', 'netstats', 'IPv6 Packet Statistics', 0),('device', 'ipsystemstats_ipv4_frag', 'netstats', 'IPv4 Fragmentation Statistics', 0),('device',  'fortigate_sessions',  'firewall',  'Active Sessions',  0), ('device',  'screenos_sessions',  'firewall',  'Active Sessions',  0), ('device',  'fdb_count',  'system',  'MAC Addresses Learnt',  0),('device', 'cras_sessions', 'firewall', 'Remote Access Sessions', 0);

doc/Installation/Installation-CentOS-6-Apache-Nginx.md

@@ -59,7 +59,7 @@ Enter the MySQL/MariaDB root password to enter the command-line interface.
 Create database.
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'<ip>'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-CentOS-7-Apache.md

@@ -13,7 +13,7 @@ mysql -uroot -p
 ```
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'localhost'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-CentOS-7-Nginx.md

@@ -13,7 +13,7 @@ mysql -uroot -p
 ```
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'localhost'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-Ubuntu-1404-Apache.md

@@ -29,7 +29,7 @@ Input the MySQL root password to enter the MySQL command-line interface.
 Create the database:
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'<ip>'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-Ubuntu-1404-Lighttpd.md

@@ -28,7 +28,7 @@ Enter the MySQL root password to enter the MySQL command-line interface.
 Create database.
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'<ip>'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-Ubuntu-1404-Nginx.md

@@ -29,7 +29,7 @@ Input the MySQL root password to enter the MySQL command-line interface.
 Create the database:
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'<ip>'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-Ubuntu-1604-Apache.md

@@ -13,7 +13,7 @@ mysql -uroot -p
 ```
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'localhost'
   IDENTIFIED BY '<password>'

doc/Installation/Installation-Ubuntu-1604-Nginx.md

@@ -13,7 +13,7 @@ mysql -uroot -p
 ```
 
 ```sql
-CREATE DATABASE librenms;
+CREATE DATABASE librenms CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 GRANT ALL PRIVILEGES ON librenms.*
   TO 'librenms'@'localhost'
   IDENTIFIED BY '<password>'

html/install.php

@@ -43,6 +43,9 @@
 // Check we can connect to MySQL DB, if not, back to stage 1 :)
 if ($stage > 1) {
     $database_link = mysqli_connect('p:'.$dbhost, $dbuser, $dbpass, $dbname, $dbport);
+    dbQuery("SET NAMES 'utf8'");
+    dbQuery("SET CHARACTER SET 'utf8'");
+    dbQuery("SET COLLATION_CONNECTION = 'utf8_unicode_ci'");
     if (mysqli_connect_error()) {
         $stage = 1;
         $msg = "Couldn't connect to the database, please check your details<br /> " . mysqli_connect_error();

includes/init.php

@@ -109,6 +109,9 @@
         die;
     }
     $database_db = mysqli_select_db($database_link, $config['db_name']);
+    dbQuery("SET NAMES 'utf8'");
+    dbQuery("SET CHARACTER SET 'utf8'");
+    dbQuery("SET COLLATION_CONNECTION = 'utf8_unicode_ci'");
 
     // pull in the database config settings
     mergedb();

sql-schema/171.sql

@@ -39,10 +39,13 @@ public static function setUpBeforeClass()
     {
         if (getenv('DBTEST')) {
             global $config;
-
             self::$sql_mode = dbFetchCell("SELECT @@global.sql_mode as sql_mode");
-            self::$db_created = dbQuery("CREATE DATABASE " . $config['db_name']);
+            dbQuery("SET NAMES 'utf8'");
+            dbQuery("SET CHARACTER SET 'utf8'");
+            dbQuery("SET COLLATION_CONNECTION = 'utf8_unicode_ci'");
+            self::$db_created = dbQuery("CREATE DATABASE " . $config['db_name'] . " CHARACTER SET utf8 COLLATE utf8_unicode_ci");
             dbQuery("SET GLOBAL sql_mode='ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'");
+            dbQuery("USE " . $config['db_name']);
             $build_base = $config['install_dir'] . '/build-base.php';
             exec($build_base, $schema);
             self::$schema = $schema;
@@ -71,4 +74,44 @@ public function testSetupDB()
             }
         }
     }
+
+    public function testCheckDBCollation()
+    {
+        global $config;
+        if (getenv('DBTEST')) {
+            $collation = dbFetchRows("SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA S WHERE schema_name = '" . $config['db_name'] . "' AND  ( DEFAULT_CHARACTER_SET_NAME != 'utf8' OR DEFAULT_COLLATION_NAME != 'utf8_unicode_ci')");
+            if (isset($collation[0])) {
+                $error = implode(' ', $collation[0]);
+            } else {
+                $error = '';
+            }
+            $this->assertEmpty($collation, 'Wrong Database Collation or Character set: ' . $error);
+        }
+    }
+
+    public function testCheckTableCollation()
+    {
+        global $config;
+        if (getenv('DBTEST')) {
+            $collation = dbFetchRows("SELECT T.TABLE_NAME, C.CHARACTER_SET_NAME, C.COLLATION_NAME FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C WHERE C.collation_name = T.table_collation AND T.table_schema = '" . $config['db_name'] . "' AND  ( C.CHARACTER_SET_NAME != 'utf8' OR C.COLLATION_NAME != 'utf8_unicode_ci' );");
+            $error = '';
+            foreach ($collation as $id => $data) {
+                $error .= implode(' ', $data) . PHP_EOL;
+            }
+            $this->assertEmpty($collation, 'Wrong Table Collation or Character set: ' . $error);
+        }
+    }
+
+    public function testCheckColumnCollation()
+    {
+        global $config;
+        if (getenv('DBTEST')) {
+            $collation = dbFetchRows("SELECT TABLE_NAME, COLUMN_NAME, CHARACTER_SET_NAME, COLLATION_NAME FROM information_schema.COLUMNS  WHERE TABLE_SCHEMA = '" . $config['db_name'] . "'  AND  ( CHARACTER_SET_NAME != 'utf8' OR COLLATION_NAME != 'utf8_unicode_ci' );");
+            $error = '';
+            foreach ($collation as $id => $data) {
+                $error .= implode(' ', $data) . PHP_EOL;
+            }
+            $this->assertEmpty($collation, 'Wrong Column Collation or Character set: ' . $error);
+        }
+    }
 }

tests/DBSetupTest.php

@@ -172,6 +172,28 @@
     print_fail("You have not set sql_mode='' in your mysql config");
 }
 
+// Test for correct character set and collation
+$collation = dbFetchRows("SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA S WHERE schema_name = '" . $config['db_name'] . "' AND  ( DEFAULT_CHARACTER_SET_NAME != 'utf8' OR DEFAULT_COLLATION_NAME != 'utf8_unicode_ci')");
+if (empty($collation) !== true) {
+    print_fail('MySQL Database collation is wrong: ' . implode(' ', $collation[0]));
+}
+$collation = dbFetchRows("SELECT T.TABLE_NAME, C.CHARACTER_SET_NAME, C.COLLATION_NAME FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C WHERE C.collation_name = T.table_collation AND T.table_schema = '" . $config['db_name'] . "' AND  ( C.CHARACTER_SET_NAME != 'utf8' OR C.COLLATION_NAME != 'utf8_unicode_ci' );");
+if (empty($collation) !== true) {
+    $error = '';
+    foreach ($collation as $id => $data) {
+        $error .= implode(' ', $data) . PHP_EOL;
+    }
+    print_fail('MySQL tables collation is wrong: ' . $error);
+}
+$collation = dbFetchRows("SELECT TABLE_NAME, COLUMN_NAME, CHARACTER_SET_NAME, COLLATION_NAME FROM information_schema.COLUMNS  WHERE TABLE_SCHEMA = '" . $config['db_name'] . "'  AND  ( CHARACTER_SET_NAME != 'utf8' OR COLLATION_NAME != 'utf8_unicode_ci' );");
+if (empty($collation) !== true) {
+    $error = '';
+    foreach ($collation as $id => $data) {
+        $error .= implode(' ', $data) . PHP_EOL;
+    }
+    print_fail('MySQL column collation is wrong: ' . $error);
+}
+
 $ini_tz = ini_get('date.timezone');
 $sh_tz = rtrim(shell_exec('date +%Z'));
 $php_tz = date('T');