diff --git a/README.markdown b/README.markdown index 2dd86f5c..dc025aa0 100644 --- a/README.markdown +++ b/README.markdown @@ -90,12 +90,12 @@ You can override any configuration setting by using the `override_config_setting mysql_servers => { 'mysql1' => { 'address' => '127.0.0.1', -         'port' => 33061, -       }, + 'port' => 33061, + }, 'mysql2' => { 'address' => '127.0.0.1', -         'port' => 33062, -       }, + 'port' => 33062, + }, ... }, mysql_users => { ... }, @@ -164,12 +164,23 @@ The username ProxySQL will use to connect to the configured mysql_servers. Defau ##### `monitor_password` The password ProxySQL will use to connect to the configured mysql_servers. Defaults to 'monitor' -##### `config_file` -The file where the ProxySQL configuration is saved. This will only be configured if `manage_config_file` is set to `true`. -Defaults to '/etc/proxysql.cnf' +##### `main_config_file` +The file where the main ProxySQL configuration is saved (datadir, admin and mysql variable). +This will only be configured if `manage_main_config_file` is set to `true`. Defaults to '/etc/proxysql.cn -##### `manage_config_file` -Determines wheter this module will configure the ProxySQL configuration file. Defaults to 'true' +##### `manage_main_config_file` +Determines wheter this module will update the ProxySQL main configuration file. Defaults to 'tru + +##### `proxy_config_file` +The file where servers, users, hostgroups, rules the ProxySQL configuration is saved. +This will only be configured if `manage_proxy_config_file` is set to `true`. Defaults to '/etc/proxysql.cn + +##### `manage_proxy_config_file` +Determines wheter this module will update the ProxySQL proxy configuration file. Defaults to 'tru + +##### `config_directory` +Path where proxy_config_file file will be stored. +Defaults to '/etc/proxysql.d ##### `mycnf_file_name` Path of the my.cnf file where the connections details for the admin interface is save. This is required for the providers to work. @@ -226,6 +237,34 @@ key utl for the yumrepo-resource in RedHat-based systems, defaults to 'https://w ##### `override_config_settings` Which configuration variables should be overriden. Hash, defaults to {} (empty hash). +##### `cluster_name` +If set, proxysql_servers with the same cluster_name will be automatically added to the same cluster and will synchronize their configuration parameters. +Defaults to '' + +##### `cluster_username` +The username ProxySQL will use to connect to the configured mysql_clusters +Defaults to 'cluster' + +##### `cluster_password` +The password ProxySQL will use to connect to the configured mysql_clusters. Defaults to 'cluster' + +##### `admin_users` +Array of users, for which .my.cnf file will be copied to their home directory. Defaults to [] + +##### `mysql_servers` +Array of mysql_servers, that will be created in ProxySQL. Defaults to undef + +##### `mysql_users` +Array of mysql_users, that will be created in ProxySQL. Defaults to undef + +##### `mysql_hostgroups` +Array of mysql_hostgroups, that will be created in ProxySQL. Defaults to undef + +##### `mysql_rules` +Array of mysql_rules, that will be created in ProxySQL. Defaults to undef + +##### `schedulers` +Array of schedulers, that will be created in ProxySQL. Defaults to undef ## Types #### proxy_global_variable @@ -243,6 +282,18 @@ Specifies wheter the resource should be immediately save to disk. Boolean, defau ##### `value` The value of the variable. +#### proxy_cluster +`proxy_cluster` manages an entry in the ProxySQL `proxysql_clusters` admin table. + +##### `name` +The name of the resource. + +##### `hostname` +Hostname of the server. Required. + +##### `port` +Port of the server. Required. Defaults to 3306. + #### proxy_mysql_replication_hostgroup `proxy_mysql_replication_hostgroup` manages an entry in the ProxySQL `mysql_replication_hostgroups` admin table. @@ -515,4 +566,4 @@ We are open to feature requests, bug reports, contributions, etc... ## Contributors -Original author: Matthias Crauwels +Original author: Matthias Crauwels \ No newline at end of file diff --git a/examples/init.pp b/examples/init.pp index 7abdb2ea..d58090f6 100644 --- a/examples/init.pp +++ b/examples/init.pp @@ -1,4 +1,35 @@ # lint:ignore:80chars +# lint:ignore:2sp_soft_tabs +# variant 1 + +class { '::proxysql': + cluster_name => 'cluster11', + mysql_servers => [ { 'db1' => { 'port' => 3306, + 'hostgroup_id' => 1, } }, + { 'db2' => { 'hostgroup_id' => 2, } }, + ], + mysql_users => [ { 'app' => { 'password' => '*92C74DFBDA5D60ABD41EFD7EB0DAE389F4646ABB', + 'default_hostgroup' => 1, } }, + { 'ro' => { 'password' => '*86935F2843252CFAAC4CE713C0D5FF80CF444F3B', + ' default_hostgroup' => 2, } }, + ], + mysql_hostgroups => [ { 'hostgroup 1' => { 'writer_hostgroup' => 1, + 'reader_hostgroup' => 2, } }, + ], + mysql_rules => [ { 'testable to test DB' => { 'rule_id' => 1, + 'match_pattern' => 'testtable', + 'replace_pattern' => 'test.newtable', + 'apply' => 1, + 'active' => 1, } }, + ], + schedulers => [ { 'test scheduler' => { 'scheduler_id' => 1, + 'active' => 0, + 'filename' => '/usr/bin/whoami', } }, + ], +} +# lint:endignore + +# variant 2 class { '::proxysql': listen_port => 3306, diff --git a/lib/puppet/provider/proxy_cluster/proxysql.rb b/lib/puppet/provider/proxy_cluster/proxysql.rb new file mode 100644 index 00000000..cd949081 --- /dev/null +++ b/lib/puppet/provider/proxy_cluster/proxysql.rb @@ -0,0 +1,128 @@ +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'proxysql')) +Puppet::Type.type(:proxy_cluster).provide(:proxysql, parent: Puppet::Provider::Proxysql) do + desc 'Manage cluster for a ProxySQL instance.' + commands mysql: 'mysql' + + def self.mysql_running + system("mysql #{defaults_file} -NBe 'SELECT 1' >out 2>&1", out: '/dev/null') + end + + # Build a property_hash containing all the discovered information about MySQL + # servers. + def self.instances + instances = [] + if mysql_running + servers = mysql([defaults_file, '-NBe', + 'SELECT `hostname`, `port` FROM `proxysql_servers`'].compact).split(%r{\n}) + + # To reduce the number of calls to MySQL we collect all the properties in + # one big swoop. + servers.each do |line| + hostname, port = line.split(%r{\t}) + query = 'SELECT `hostname`, `port`, `weight`, `comment`' + query << ' FROM `proxysql_servers`' + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + + @hostname, @port, @weight, @comment = mysql([defaults_file, '-NBe', query].compact).chomp.split(%r{\t}) + name = "#{hostname}:#{port}" + + instances << new( + name: name, + ensure: :present, + hostname: @hostname, + port: @port, + weight: @weight, + comment: @comment + ) + end + end + instances + end + + # We iterate over each proxy_mysql_server entry in the catalog and compare it against + # the contents of the property_hash generated by self.instances + def self.prefetch(resources) + servers = instances + resources.keys.each do |name| + provider = servers.find { |server| server.name == name } + resources[name].provider = provider if provider + end + end + + def create + _name = @resource[:name] + hostname = @resource.value(:hostname) + port = @resource.value(:port) || 6032 + weight = @resource.value(:weight) || 0 + comment = @resource.value(:comment) || '' + + query = 'INSERT INTO proxysql_servers (`hostname`, `port`, `weight`, `comment`)' + query << " VALUES ('#{hostname}', #{port}, #{weight}, '#{comment}')" + mysql([defaults_file, '-e', query].compact) + @property_hash[:ensure] = :present + + exists? ? (return true) : (return false) + end + + def destroy + hostname = @property_hash[:hostname] + port = @property_hash[:port] + query = 'DELETE FROM `proxysql_servers`' + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + mysql([defaults_file, '-e', query].compact) + + @property_hash.clear + exists? ? (return false) : (return true) + end + + def exists? + @property_hash[:ensure] == :present || false + end + + def initialize(value = {}) + super(value) + @property_flush = {} + end + + def flush + update_server(@property_flush) if @property_flush + @property_hash.clear + + load_to_runtime = @resource[:load_to_runtime] + mysql([defaults_file, '-NBe', 'LOAD PROXYSQL SERVERS TO RUNTIME'].compact) if load_to_runtime == :true + + save_to_disk = @resource[:save_to_disk] + mysql([defaults_file, '-NBe', 'SAVE PROXYSQL SERVERS TO DISK'].compact) if save_to_disk == :true + end + + def update_server(properties) + hostname = @resource.value(:hostname) + port = @resource.value(:port) + + return false if properties.empty? + + values = [] + properties.each do |field, value| + values.push("`#{field}` = '#{value}'") + end + + query = 'UPDATE proxysql_servers SET ' + query << values.join(', ') + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + mysql([defaults_file, '-e', query].compact) + + @property_hash.clear + exists? ? (return false) : (return true) + end + + # Generates method for all properties of the property_hash + mk_resource_methods + + def weight=(value) + @property_flush[:weight] = value + end + + def comment=(value) + @property_flush[:comment] = value + end +end diff --git a/lib/puppet/provider/proxy_mysql_server_no_hostgroup/proxysql.rb b/lib/puppet/provider/proxy_mysql_server_no_hostgroup/proxysql.rb new file mode 100644 index 00000000..feb336e0 --- /dev/null +++ b/lib/puppet/provider/proxy_mysql_server_no_hostgroup/proxysql.rb @@ -0,0 +1,166 @@ +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'proxysql')) +Puppet::Type.type(:proxy_mysql_server_no_hostgroup).provide(:proxysql, parent: Puppet::Provider::Proxysql) do + desc 'Manage servers for a ProxySQL instance.' + commands mysql: 'mysql' + + # Build a property_hash containing all the discovered information about MySQL + # servers. + def self.instances + instances = [] + servers = mysql([defaults_file, '-NBe', + 'SELECT `hostname`, `port` FROM `mysql_servers`'].compact).split(%r{\n}) + + # To reduce the number of calls to MySQL we collect all the properties in + # one big swoop. + servers.each do |line| + hostname, port = line.split(%r{\t}) + query = 'SELECT `hostname`, `port`, `hostgroup_id`, `status`, `weight`, `compression`, ' + query << ' `max_connections`, `max_replication_lag`, `use_ssl`, `max_latency_ms`, `comment` ' + query << ' FROM `mysql_servers`' + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + + @hostname, @port, @hostgroup_id, @status, @weight, @compression, + @max_connections, @max_replication_lag, @use_ssl, @max_latency_ms, + @comment = mysql([defaults_file, '-NBe', query].compact).chomp.split(%r{\t}) + name = "#{hostname}:#{port}" + + instances << new( + name: name, + ensure: :present, + hostname: @hostname, + port: @port, + hostgroup_id: @hostgroup_id, + status: @status, + weight: @weight, + compression: @compression, + max_connections: @max_connections, + max_replication_lag: @max_replication_lag, + use_ssl: @use_ssl, + max_latency_ms: @max_latency_ms, + comment: @comment + ) + end + instances + end + + # We iterate over each proxy_mysql_server entry in the catalog and compare it against + # the contents of the property_hash generated by self.instances + def self.prefetch(resources) + servers = instances + resources.keys.each do |name| + provider = servers.find { |server| server.name == name } + resources[name].provider = provider if provider + end + end + + def create + _name = @resource[:name] + hostname = @resource.value(:hostname) + port = @resource.value(:port) || 3306 + hostgroup_id = @resource.value(:hostgroup_id) || 0 + status = @resource.value(:status) || 'ONLINE' + weight = @resource.value(:weight) || 1 + compression = @resource.value(:compression) || 0 + max_connections = @resource.value(:max_connections) || 1000 + max_replication_lag = @resource.value(:max_replication_lag) || 0 + use_ssl = @resource.value(:use_ssl) || 0 + max_latency_ms = @resource.value(:max_latency_ms) || 0 + comment = @resource.value(:comment) || '' + + query = 'INSERT INTO mysql_servers (`hostname`, `port`, `hostgroup_id`, `status`, `weight`, `compression`, ' + query << ' `max_connections`, `max_replication_lag`, `use_ssl`, `max_latency_ms`, `comment`)' + query << " VALUES ('#{hostname}', #{port}, #{hostgroup_id}, '#{status}', #{weight}, #{compression}, " + query << " #{max_connections}, #{max_replication_lag}, #{use_ssl}, #{max_latency_ms}, '#{comment}')" + mysql([defaults_file, '-e', query].compact) + @property_hash[:ensure] = :present + + exists? ? (return true) : (return false) + end + + def destroy + hostname = @resource.value(:hostname) + port = @resource.value(:port) + + query = 'DELETE FROM `mysql_servers`' + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + mysql([defaults_file, '-e', query].compact) + + @property_hash.clear + exists? ? (return false) : (return true) + end + + def exists? + @property_hash[:ensure] == :present || false + end + + def initialize(value = {}) + super(value) + @property_flush = {} + end + + def flush + update_server(@property_flush) if @property_flush + @property_hash.clear + + load_to_runtime = @resource[:load_to_runtime] + mysql([defaults_file, '-NBe', 'LOAD MYSQL SERVERS TO RUNTIME'].compact) if load_to_runtime == :true + + save_to_disk = @resource[:save_to_disk] + mysql([defaults_file, '-NBe', 'SAVE MYSQL SERVERS TO DISK'].compact) if save_to_disk == :true + end + + def update_server(properties) + hostname = @resource.value(:hostname) + port = @resource.value(:port) + + return false if properties.empty? + + values = [] + properties.each do |field, value| + values.push("`#{field}` = '#{value}'") + end + + query = 'UPDATE mysql_servers SET ' + query << values.join(', ') + query << " WHERE `hostname` = '#{hostname}' AND `port` = #{port}" + mysql([defaults_file, '-e', query].compact) + + @property_hash.clear + exists? ? (return false) : (return true) + end + + # Generates method for all properties of the property_hash + mk_resource_methods + + def status=(value) + @property_flush[:status] = value + end + + def weight=(value) + @property_flush[:weight] = value + end + + def compression=(value) + @property_flush[:compression] = value + end + + def max_connections=(value) + @property_flush[:max_connections] = value + end + + def max_replication_lag=(value) + @property_flush[:max_replication_lag] = value + end + + def use_ssl=(value) + @property_flush[:use_ssl] = value + end + + def max_latency_ms=(value) + @property_flush[:max_latency_ms] = value + end + + def comment=(value) + @property_flush[:comment] = value + end +end diff --git a/lib/puppet/type/proxy_cluster.rb b/lib/puppet/type/proxy_cluster.rb new file mode 100644 index 00000000..eb1926d3 --- /dev/null +++ b/lib/puppet/type/proxy_cluster.rb @@ -0,0 +1,52 @@ +# This has to be a separate type to enable collecting +Puppet::Type.newtype(:proxy_cluster) do + @doc = 'Manage a ProxySQL cluster.' + + ensurable + + autorequire(:file) { '/root/.my.cnf' } + autorequire(:class) { 'mysql::client' } + autorequire(:service) { 'proxysql' } + + validate do + raise('name parameter is required.') if (self[:ensure] == :present) && self[:name].nil? + raise('hostname parameter is required.') if (self[:ensure] == :present) && self[:hostname].nil? + raise('port parameter is required.') if (self[:ensure] == :present) && self[:port].nil? + end + + newparam(:name, namevar: true) do + desc 'name for cluster to manage.' + end + + newparam(:load_to_runtime) do + desc 'Load this entry to the active runtime.' + defaultto :true + newvalues(:true, :false) + end + + newparam(:save_to_disk) do + desc 'Perist this entry to the disk.' + defaultto :true + newvalues(:true, :false) + end + + newproperty(:hostname) do + desc 'The hostname of the server.' + newvalue(%r{\w+}) + end + + newproperty(:port) do + desc 'The port of the server.' + newvalue(%r{\d+}) + end + + newproperty(:weight) do + desc 'Currently unused, but in the roadmap for future enhancements.' + newvalue(%r{\d+}) + end + + newproperty(:comment) do + desc 'free form comment field.' + newvalue(%r{[\w+]}) + end +end diff --git a/lib/puppet/type/proxy_mysql_server_no_hostgroup.rb b/lib/puppet/type/proxy_mysql_server_no_hostgroup.rb new file mode 100644 index 00000000..c082d2f7 --- /dev/null +++ b/lib/puppet/type/proxy_mysql_server_no_hostgroup.rb @@ -0,0 +1,91 @@ +# This has to be a separate type to enable collecting +Puppet::Type.newtype(:proxy_mysql_server_no_hostgroup) do + @doc = 'Manage a ProxySQL mysql_server.' + + ensurable + + autorequire(:file) { '/root/.my.cnf' } + autorequire(:class) { 'mysql::client' } + autorequire(:service) { 'proxysql' } + + validate do + raise('hostname parameter is required.') if (self[:ensure] == :present) && self[:hostname].nil? + raise('port parameter is required.') if (self[:ensure] == :present) && self[:port].nil? + raise('hostgroup_id parameter is required.') if (self[:ensure] == :present) && self[:hostgroup_id].nil? + raise('name must match hostname and port') if self[:name] != "#{self[:hostname]}:#{self[:port]}" + end + + newparam(:name, namevar: true) do + desc 'name for server to manage.' + end + + newparam(:load_to_runtime) do + desc 'Load this entry to the active runtime.' + defaultto :true + newvalues(:true, :false) + end + + newparam(:save_to_disk) do + desc 'Perist this entry to the disk.' + defaultto :true + newvalues(:true, :false) + end + + newparam(:hostgroup_id) do + desc 'The hostgroup of the server.' + defaultto 0 + newvalues(%r{\d+}) + end + + newproperty(:hostname) do + desc 'The hostname of the server.' + defaultto :localhost + newvalue(%r{\w+}) + end + + newproperty(:port) do + desc 'The port of the server.' + defaultto 3306 + newvalue(%r{\d+}) + end + + newproperty(:status) do + desc 'Server status.' + newvalues(:ONLINE, :SHUNNED, :OFFLINE_SOFT, :OFFLINE_HARD) + end + + newproperty(:weight) do + desc 'the bigger the weight of a server relative to other weights, the higher the probability of the server to be chosen from a hostgroup' + newvalue(%r{\d+}) + end + + newproperty(:compression) do + desc 'if the value is greater than 0, new connections to that server will use compression' + newvalue(%r{\d+}) + end + + newproperty(:max_connections) do + desc 'the maximum number of connections ProxySQL will open to this backend server. Even though this server will have the highest weight, no new connections will be opened to it once this limit is hit. Please ensure that the backend is configured with a correct value of max_connections to avoid that ProxySQL will try to go beyond that limit' + newvalue(%r{\d+}) + end + + newproperty(:max_replication_lag) do + desc 'if greater and 0, ProxySQL will reguarly monitor replication lag and if it goes beyond such threshold it will temporary shun the host until replication catch ups' + newvalue(%r{\d+}) + end + + newproperty(:use_ssl) do + desc 'if set to 1, connections to the backend will use SSL' + newvalue(%r{[01]}) + end + + newproperty(:max_latency_ms) do + desc 'ping time is regularly monitored. If a host has a ping time greater than max_latency_ms it is excluded from the connection pool (although the server stays ONLINE)' + newvalue(%r{[\d+]}) + end + + newproperty(:comment) do + desc 'text field that can be used for any purposed defined by the user. Could be a description of what the host stores, a reminder of when the host was added or disabled, or a JSON processed by some checker script.' + newvalue(%r{[\w+]}) + end +end diff --git a/manifests/admin_credentials.pp b/manifests/admin_credentials.pp index bbf08e85..3bf1a989 100644 --- a/manifests/admin_credentials.pp +++ b/manifests/admin_credentials.pp @@ -6,8 +6,10 @@ if $proxysql::manage_mycnf_file { $mycnf_file_name = $proxysql::mycnf_file_name + $admin_users = $proxysql::admin_users $admin_credentials = $proxysql::config_settings['admin_variables']['admin_credentials'] $admin_interfaces = $proxysql::config_settings['admin_variables']['mysql_ifaces'] + # lint:ignore:140chars exec { 'proxysql-admin-credentials': command => "/usr/bin/mysql --defaults-extra-file=${mycnf_file_name} --execute=\" SET admin-admin_credentials = '${admin_credentials}'; \ @@ -23,14 +25,26 @@ ", before => File['root-mycnf-file'], } + # lint:endignore file { 'root-mycnf-file': - ensure => file, - path => $proxysql::mycnf_file_name, - content => template('proxysql/my.cnf.erb'), - owner => $proxysql::sys_owner, - group => $proxysql::sys_group, - mode => '0400', + ensure => file, + path => $proxysql::mycnf_file_name, + content => template('proxysql/my.cnf.erb'), + owner => 'root', + group => 'root', + mode => '0640', + } + + $admin_users.each |String $user| { + file { "${::user}-mycnf-file": + ensure => file, + path => "/home/${user}/.my.cnf", + content => template('proxysql/my.cnf.erb'), + owner => $user, + group => 'root', + mode => '0640', + } } } diff --git a/manifests/cluster.pp b/manifests/cluster.pp new file mode 100644 index 00000000..c70db8fa --- /dev/null +++ b/manifests/cluster.pp @@ -0,0 +1,21 @@ +# == Class proxysql::cluster +# +# This class is called from proxysql for cluster config. +# +class proxysql::cluster { + + if $proxysql::cluster_name != '' { + resources { 'proxy_cluster': + purge => true, + } + $query = "resources[parameters] {type = 'Class' and title = 'Proxysql' and parameters.cluster_name = '${proxysql::cluster_name}'}" + $nodes = puppetdb_query($query).map | $hash | { $hash['parameters']['node_name'] } + + $nodes.each |String $node| { + proxy_cluster { $node: + hostname => "${split($node, ':')[0]}", + port => split($node, ':')[1], + } + } + } +} diff --git a/manifests/config.pp b/manifests/config.pp index 221c2f25..4b8d96a5 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -5,16 +5,51 @@ class proxysql::config { $config_settings = $::proxysql::config_settings - if $proxysql::manage_config_file { - file { 'proxysql-config-file': - ensure => file, - path => $proxysql::config_file, - content => template('proxysql/proxysql.cnf.erb'), - mode => '0640', - owner => $proxysql::sys_owner, - group => $proxysql::sys_group, - selinux_ignore_defaults => true, - } + + group { $::proxysql::sys_group: + ensure => 'present', + } + + user { $::proxysql::sys_owner: + ensure => 'present', + groups => $::proxysql::sys_group, + } + + file { 'proxysql-datadir': + ensure => directory, + path => $proxysql::datadir, + owner => $proxysql::sys_owner, + group => $proxysql::sys_group, + mode => '0600', + } + + file { 'proxysql-config-directory': + ensure => directory, + path => $proxysql::config_directory, + mode => '0640', + owner => $proxysql::sys_owner, + group => $proxysql::sys_group, + } + + file { 'proxysql-main-config-file': + ensure => file, + notify => Service[$proxysql::service_name], + path => $proxysql::main_config_file, + content => template('proxysql/proxysql.cnf.erb'), + mode => '0640', + owner => $proxysql::sys_owner, + group => $proxysql::sys_group, + replace => $proxysql::manage_main_config_file, + } + + file { 'proxysql-proxy-config-file': + ensure => file, + path => "${proxysql::config_directory}/${proxysql::proxy_config_file}", + content => template('proxysql/proxysql_proxy.cnf.erb'), + mode => '0640', + owner => $proxysql::sys_owner, + group => $proxysql::sys_group, + replace => $proxysql::manage_proxy_config_file, } } diff --git a/manifests/configure.pp b/manifests/configure.pp new file mode 100644 index 00000000..30c73299 --- /dev/null +++ b/manifests/configure.pp @@ -0,0 +1,86 @@ +# == Class proxysql::configure +# +# This class is called from proxysql for all proxy configuration. +# +class proxysql::configure { + + if $proxysql::mysql_servers { + $proxysql::mysql_servers.each |$server| { + $server.each |$k,$v| { + $hostname = $k + $port = $server[$k][port] ? { undef => 3306, + default => $server[$k][port], } + $hostgroup_id = $server[$k][hostgroup_id] + + if $proxysql::manage_hostgroup_for_servers { + proxy_mysql_server { "${hostname}:${port}-${hostgroup_id}": + hostname => $hostname, + * => $server[$k], + } + } else { + proxy_mysql_server_no_hostgroup { "${hostname}:${port}": + hostname => $hostname, + * => $server[$k], + } + } + } + } + } + + if $proxysql::mysql_users { + $proxysql::mysql_users.each |$user| { + $user.each |$k,$v| { + $username = $k + + proxy_mysql_user { $username: + * => $user[$k], + } + } + } + } + + if $proxysql::mysql_hostgroups { + $proxysql::mysql_hostgroups.each |$hostgroup| { + $hostgroup.each |$k,$v| { + $comment = $k + $reader = $hostgroup[$k][reader] + $writer = $hostgroup[$k][writer] + + proxy_mysql_replication_hostgroup { "${writer}-${reader}": + writer_hostgroup => $writer, + reader_hostgroup => $reader, + comment => $k, + } + } + } + } + + if $proxysql::mysql_rules { + $proxysql::mysql_rules.each |$rule| { + $rule.each |$k,$v| { + $comment = $k + $rule_id = $rule[$k][rule_id] + + proxy_mysql_query_rule { "mysql_query_rule-${rule_id}": + comment => $k, + * => $rule[$k], + } + } + } + } + + if $proxysql::schedulers { + $proxysql::schedulers.each |$scheduler| { + $scheduler.each |$k,$v| { + $comment = $k + $scheduler_id = $scheduler[$k][scheduler_id] + + proxy_scheduler { "scheduler-${scheduler_id}": + comment => $k, + * => $scheduler[$k], + } + } + } + } +} + diff --git a/manifests/init.pp b/manifests/init.pp index d7d9a8a9..b02075dc 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -52,12 +52,23 @@ # * `monitor_password` # The password ProxySQL will use to connect to the configured mysql_servers. Defaults to 'monitor' # -# * `config_file` -# The file where the ProxySQL configuration is saved. This will only be configured if `manage_config_file` is set to `true`. -# Defaults to '/etc/proxysql.cnf' +# * `main_config_file` +# The file where the main ProxySQL configuration is saved (datadir, admin and mysql variable). +# This will only be configured if `manage_main_config_file` is set to `true`. Defaults to '/etc/proxysql.cnf' # -# * `manage_config_file` -# Determines wheter this module will configure the ProxySQL configuration file. Defaults to 'true' +# * `manage_main_config_file` +# Determines wheter this module will update the ProxySQL main configuration file. Defaults to 'true' +# +# * `proxy_config_file` +# The file where servers, users, hostgroups, rules the ProxySQL configuration is saved. +# This will only be configured if `manage_proxy_config_file` is set to `true`. Defaults to '/etc/proxysql.cnf' +# +# * `manage_proxy_config_file` +# Determines wheter this module will update the ProxySQL proxy configuration file. Defaults to 'true' +# +# * `config_directory` +# Path where proxy_config_file file will be stored. +# Defaults to '/etc/proxysql.d/' # # * `mycnf_file_name` # Path of the my.cnf file where the connections details for the admin interface is save. This is required for the providers to work. @@ -67,6 +78,10 @@ # Determines wheter this module will configure the my.cnf file to connect to the admin interface. # This is required for the providers to work. Defaults to 'true' # +# * `manage_hostgroup_for_servers` +# Determines wheter this module will manage hostgroup_id for mysql_servers. +# If false - it will skip difference in this value between manifest and defined in ProxySQL. Defaults to 'true' +# # * `restart` # Determines wheter this module will restart ProxySQL after reconfiguring the config file. Defaults to 'false' # @@ -84,7 +99,7 @@ # to `apt::source`. Defaults to {}. # # * `package_source` -# location ot the proxysql package for the `package_provider`. Default to 'https://www.percona.com/redir/downloads/proxysql/proxysql-1.3.2/binary/redhat/6/x86_64/proxysql-1.3.2-1.1.x86_64.rpm' +# location ot the proxysql package for the `package_provider`. Default to undef # # * `package_provider` # provider for package-resource. defaults to `dpkg` for debian-based, `rpm` for redhat base or undef for others @@ -110,7 +125,37 @@ # * `override_config_settings` # Which configuration variables should be overriden. Hash, defaults to {} (empty hash). # +# * `cluster_name` +# If set, proxysql_servers with the same cluster_name will be automatically added to the same cluster and will +# synchronize their configuration parameters. Defaults to '' +# +# * `cluster_username` +# The username ProxySQL will use to connect to the configured mysql_clusters +# Defaults to 'cluster' +# +# * `cluster_password` +# The password ProxySQL will use to connect to the configured mysql_clusters. Defaults to 'cluster' +# +# * `admin_users` +# Array of users, for which .my.cnf file will be copied to their home directory. Defaults to [] +# +# * `mysql_servers` +# Array of mysql_servers, that will be created in ProxySQL. Defaults to undef +# +# * `mysql_users` +# Array of mysql_users, that will be created in ProxySQL. Defaults to undef +# +# * `mysql_hostgroups` +# Array of mysql_hostgroups, that will be created in ProxySQL. Defaults to undef +# +# * `mysql_rules` +# Array of mysql_rules, that will be created in ProxySQL. Defaults to undef +# +# * `schedulers` +# Array of schedulers, that will be created in ProxySQL. Defaults to undef +# class proxysql ( + String $cluster_name = '', String $package_name = $::proxysql::params::package_name, String $package_ensure = $::proxysql::params::package_ensure, Array[String] $package_install_options = $::proxysql::params::package_install_options, @@ -132,8 +177,11 @@ String $monitor_username = $::proxysql::params::monitor_username, Sensitive[String] $monitor_password = $::proxysql::params::monitor_password, - String $config_file = $::proxysql::params::config_file, - Boolean $manage_config_file = $::proxysql::params::manage_config_file, + String $main_config_file = $::proxysql::params::main_config_file, + Boolean $manage_main_config_file = $::proxysql::params::manage_main_config_file, + String $config_directory = $::proxysql::params::config_directory, + String $proxy_config_file = $::proxysql::params::proxy_config_file, + Boolean $manage_proxy_config_file = $::proxysql::params::manage_proxy_config_file, String $mycnf_file_name = $::proxysql::params::mycnf_file_name, Boolean $manage_mycnf_file = $::proxysql::params::manage_mycnf_file, @@ -142,12 +190,13 @@ Boolean $load_to_runtime = $::proxysql::params::load_to_runtime, Boolean $save_to_disk = $::proxysql::params::save_to_disk, + Boolean $manage_hostgroup_for_servers = $::proxysql::params::manage_hostgroup_for_servers, - Boolean $manage_repo = true, + Boolean $manage_repo = $::proxysql::params::manage_repo, Hash $repo = {}, - String $package_source = $::proxysql::params::package_source, - String $package_provider = $::proxysql::params::package_provider, + Optional[String] $package_source = '', + String $package_provider = $::proxysql::params::package_provider, String $sys_owner = $::proxysql::params::sys_owner, String $sys_group = $::proxysql::params::sys_group, @@ -157,10 +206,23 @@ String $rpm_repo = $::proxysql::params::rpm_repo, String $rpm_repo_key = $::proxysql::params::rpm_repo_key, + String $cluster_username = $::proxysql::params::cluster_username, + Sensitive[String] $cluster_password = $::proxysql::params::cluster_password, + Hash $override_config_settings = {}, + + Optional[Array[String]] $admin_users = [], + + Optional[Proxysql::Server] $mysql_servers = undef, + Optional[Proxysql::User] $mysql_users = undef, + Optional[Proxysql::Hostgroup] $mysql_hostgroups = undef, + Optional[Proxysql::Rule] $mysql_rules = undef, + Optional[Proxysql::Scheduler] $schedulers = undef, + + String $node_name = "${::fqdn}:${admin_listen_port}" ) inherits ::proxysql::params { - # lint:ignore:80chars +# lint:ignore:80chars $settings = { datadir => $datadir, admin_variables => { @@ -170,18 +232,34 @@ mysql_variables => { interfaces => "${listen_ip}:${listen_port};${listen_socket}", monitor_username => $monitor_username, - monitor_password => $monitor_password.unwrap, + monitor_password => "${monitor_password.unwrap}", }, } - $config_settings = deep_merge($proxysql::params::config_settings, $settings, $override_config_settings) + + if $cluster_name { + $settings_cluster = { + admin_variables => { + admin_credentials => "${admin_username}:${admin_password.unwrap};${cluster_username}:${cluster_password.unwrap}", + cluster_username => $cluster_username, + cluster_password => "${cluster_password.unwrap}", + }, + } + } + + $settings_result = deep_merge($settings, $settings_cluster) + + $config_settings = deep_merge($proxysql::params::config_settings, $settings_result, $override_config_settings) # lint:endignore anchor { '::proxysql::begin': } -> class { '::proxysql::repo':} - -> class { '::proxysql::install':} -> class { '::proxysql::config':} + -> class { '::proxysql::install':} -> class { '::proxysql::service':} -> class { '::proxysql::admin_credentials':} + -> class { '::proxysql::reload_config':} + -> class { '::proxysql::cluster':} + -> class { '::proxysql::configure':} -> anchor { '::proxysql::end': } Class['::proxysql::install'] diff --git a/manifests/install.pp b/manifests/install.pp index fbe691ed..74b3ffb5 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -18,16 +18,9 @@ } } - file { 'proxysql-datadir': - ensure => directory, - path => $::proxysql::datadir, - owner => $::proxysql::sys_owner, - group => $::proxysql::sys_group, - mode => '0600', - } - class { '::mysql::client': + package_name => 'mysql-client', + package_ensure => 'present', bindings_enable => false, } - } diff --git a/manifests/params.pp b/manifests/params.pp index 7e5fff2c..e0963363 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -8,9 +8,6 @@ $package_ensure = 'installed' $package_install_options = [] - # Check your version in https://github.com/sysown/proxysql/releases - $package_source = 'https://www.percona.com/redir/downloads/proxysql/proxysql-1.3.2/binary/redhat/6/x86_64/proxysql-1.3.2-1.1.x86_64.rpm' - $service_name = 'proxysql' $service_ensure = 'running' @@ -50,36 +47,46 @@ } } - $monitor_username = 'monitor' $monitor_password = Sensitive('monitor') $datadir = '/var/lib/proxysql' - $config_file = '/etc/proxysql.cnf' - $manage_config_file = true + $main_config_file = '/etc/proxysql.cnf' + $manage_main_config_file = true + $config_directory = '/etc/proxysql.d/' + $proxy_config_file = 'proxysql_proxy.cnf' + $manage_proxy_config_file = true $mycnf_file_name = '/root/.my.cnf' $manage_mycnf_file = true $restart = false - $load_to_runtime = true - $save_to_disk = true + $manage_repo = true + + $load_to_runtime = true + $save_to_disk = true + $manage_hostgroup_for_servers = true $rpm_repo_name = 'percona_repo' $rpm_repo_descr = 'percona_repo_contains_proxysql' $rpm_repo = 'http://repo.percona.com/release/$releasever/RPMS/$basearch' $rpm_repo_key = 'https://www.percona.com/downloads/RPM-GPG-KEY-percona' + $cluster_username = 'cluster' + $cluster_password = Sensitive('cluster') + $config_settings = { datadir => $datadir, admin_variables => { - admin_credentials => "${admin_username}:${admin_password}", + admin_credentials => "${admin_username}:${admin_password};", mysql_ifaces => "${admin_listen_ip}:${admin_listen_port};${admin_listen_socket}", }, mysql_variables => { interfaces => "${listen_ip}:${listen_port};${listen_socket}", + monitor_username => $monitor_username, + monitor_password => $monitor_password, }, mysql_servers => {}, mysql_users => {}, diff --git a/manifests/reload_config.pp b/manifests/reload_config.pp new file mode 100644 index 00000000..a84e997e --- /dev/null +++ b/manifests/reload_config.pp @@ -0,0 +1,21 @@ +# == Class proxysql::reload_config +# +# This class is called from proxysql to update config if it changed. +# +class proxysql::reload_config { + + $mycnf_file_name = $proxysql::mycnf_file_name + exec {'reload-config': + command => "/usr/bin/mysql --defaults-extra-file=${mycnf_file_name} --execute=\" + LOAD ADMIN VARIABLES FROM CONFIG; \ + LOAD ADMIN VARIABLES TO RUNTIME; \ + SAVE ADMIN VARIABLES TO DISK; \ + LOAD MYSQL VARIABLES FROM CONFIG; \ + LOAD MYSQL VARIABLES TO RUNTIME; \ + SAVE MYSQL VARIABLES TO DISK; \" + ", + subscribe => [ File['proxysql-main-config-file'], File['proxysql-proxy-config-file'] ], + require => [ Service[$::proxysql::service_name] , File['root-mycnf-file'] ], + refreshonly => true, + } +} diff --git a/manifests/service.pp b/manifests/service.pp index 6c9ccf22..efa15e0d 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -5,8 +5,8 @@ # class proxysql::service { - if $::proxysql::manage_config_file { - $service_require = File['proxysql-config-file'] + if $::proxysql::manage_main_config_file { + $service_require = [ File['proxysql-main-config-file'], File['proxysql-proxy-config-file'] ] } else { $service_require = undef } diff --git a/spec/classes/proxysql_spec.rb b/spec/classes/proxysql_spec.rb index b88f499c..e3659b28 100644 --- a/spec/classes/proxysql_spec.rb +++ b/spec/classes/proxysql_spec.rb @@ -13,14 +13,19 @@ it { is_expected.to compile.with_all_deps } it { is_expected.to contain_class('proxysql::params') } - it { is_expected.to contain_class('proxysql::repo').that_comes_before('Class[proxysql::install]') } - it { is_expected.to contain_class('proxysql::install').that_comes_before('Class[proxysql::config]') } - it { is_expected.to contain_class('proxysql::config').that_comes_before('Class[proxysql::service]') } - it { is_expected.to contain_class('proxysql::service').that_subscribes_to('Class[proxysql::install]') } it { is_expected.to contain_anchor('::proxysql::begin').that_comes_before('Class[proxysql::repo]') } + it { is_expected.to contain_class('proxysql::repo').that_comes_before('Class[proxysql::config]') } + it { is_expected.to contain_class('proxysql::config').that_comes_before('Class[proxysql::install]') } + it { is_expected.to contain_class('proxysql::install').that_comes_before('Class[proxysql::service]') } + it { is_expected.to contain_class('proxysql::service').that_comes_before('Class[proxysql::admin_credentials]') } + it { is_expected.to contain_class('proxysql::admin_credentials').that_comes_before('Class[proxysql::reload_config]') } + it { is_expected.to contain_class('proxysql::reload_config').that_comes_before('Class[proxysql::cluster]') } + it { is_expected.to contain_class('proxysql::cluster').that_comes_before('Class[proxysql::configure]') } + it { is_expected.to contain_class('proxysql::configure').that_comes_before('Anchor[::proxysql::end]') } it { is_expected.to contain_anchor('::proxysql::end') } - it { is_expected.to contain_class('proxysql::service').that_comes_before('Anchor[::proxysql::end]') } + + it { is_expected.to contain_class('proxysql::service').that_subscribes_to('Class[proxysql::install]') } it { is_expected.to contain_class('proxysql::install').that_notifies('Class[proxysql::service]') } @@ -47,11 +52,11 @@ end it do - is_expected.to contain_file('proxysql-config-file').with(ensure: 'file', - owner: sys_user, - group: sys_group, - mode: '0640', - path: '/etc/proxysql.cnf') + is_expected.to contain_file('proxysql-main-config-file').with(ensure: 'file', + owner: sys_user, + group: sys_group, + mode: '0640', + path: '/etc/proxysql.cnf') end it do @@ -64,9 +69,9 @@ it do is_expected.to contain_file('root-mycnf-file').with(ensure: 'file', - owner: sys_user, - group: sys_group, - mode: '0400', + owner: 'root', + group: 'root', + mode: '0640', path: '/root/.my.cnf') end diff --git a/templates/proxysql.cnf.erb b/templates/proxysql.cnf.erb index 60eaf374..f111af56 100644 --- a/templates/proxysql.cnf.erb +++ b/templates/proxysql.cnf.erb @@ -39,28 +39,6 @@ end -%> } -<% - else --%> -<%= k %> = ( -<% - v.each do |ki, vi| - if vi.is_a?(Hash) - hash_handler.call(ki, vi) - else - default_handler.call(ki, vi) - end - - if v.keys.index(ki) < v.size - 1 --%> -, -<% - end - end --%> - -) - <% end else @@ -68,3 +46,5 @@ end end end -%> + +@include "/etc/proxysql.d/proxysql_proxy.cnf" diff --git a/templates/proxysql_proxy.cnf.erb b/templates/proxysql_proxy.cnf.erb new file mode 100644 index 00000000..5081d64d --- /dev/null +++ b/templates/proxysql_proxy.cnf.erb @@ -0,0 +1,61 @@ +### MANAGED BY CONSUL ### + +<% +default_handler = Proc.new do |k, v| + if v.is_a?(Integer) || v.is_a?(TrueClass) || v.is_a?(FalseClass) +-%> + <%= k %> = <%= v %> +<% + else +-%> + <%= k %> = "<%= v %>" +<% + end +end + +hash_handler = Proc.new do |k, v| + if v.is_a?(Hash) +-%> + { +<% + v.map do |ki, vi| + hash_handler.call(ki, vi) + end +-%> + }<% + else + default_handler.call(k, v) + end +end + +@config_settings.map do |k, v| + if v.is_a?(Hash) + if k == 'admin_variables' || k == 'mysql_variables' +-%> +<% + else +-%> +<%= k %> = ( +<% + v.each do |ki, vi| + if vi.is_a?(Hash) + hash_handler.call(ki, vi) + else + default_handler.call(ki, vi) + end + + if v.keys.index(ki) < v.size - 1 +-%> +, +<% + end + end +-%> + +) + +<% + end + end +end +-%> diff --git a/types/hostgroup.pp b/types/hostgroup.pp new file mode 100644 index 00000000..b24128e6 --- /dev/null +++ b/types/hostgroup.pp @@ -0,0 +1,4 @@ +# lint:ignore:2sp_soft_tabs +type Proxysql::Hostgroup = Array[Hash[String, Struct[{ writer => Integer, + reader => Integer, }],1,1]] +# lint:endignore diff --git a/types/rule.pp b/types/rule.pp new file mode 100644 index 00000000..4aa3787b --- /dev/null +++ b/types/rule.pp @@ -0,0 +1,27 @@ +# lint:ignore:2sp_soft_tabs +type Proxysql::Rule = Array[Hash[String, Struct[{ rule_id => Integer, + active => Integer, + Optional[username] => String[1], + Optional[schemaname] => String[1], + Optional[flag_in] => Integer, + Optional[flag_out] => Integer, + Optional[apply] => Integer, + Optional[client_addr] => String[1], + Optional[proxy_addr] => String[1], + Optional[proxy_port] => Integer, + Optional[digest] => String[1], + Optional[match_digest] => String[1], + Optional[match_pattern] => String[1], + Optional[replace_pattern] => String[1], + Optional[negate_match_pattern] => Boolean, + Optional[destination_hostgroup] => Integer, + Optional[cache_ttl] => Integer, + Optional[reconnect] => Boolean, + Optional[timeout] => Integer, + Optional[retries] => Integer, + Optional[delay] => Integer, + Optional[error_msg] => String[1], + Optional[log] => Boolean, + Optional[mirror_hostgroup] => Integer, + Optional[mirror_flag_out] => Integer, }],1,1]] +# lint:endignore diff --git a/types/scheduler.pp b/types/scheduler.pp new file mode 100644 index 00000000..fb47907c --- /dev/null +++ b/types/scheduler.pp @@ -0,0 +1,11 @@ +# lint:ignore:2sp_soft_tabs +type Proxysql::Scheduler = Array[Hash[String, Struct[{ scheduler_id => Integer, + active => Integer, + Optional[interval_ms] => Integer, + filename => String[1], + Optional[arg1] => String[1], + Optional[arg2] => String[1], + Optional[arg3] => String[1], + Optional[arg4] => String[1], + Optional[arg5] => String[1] }],1,1]] +# lint:endignore diff --git a/types/server.pp b/types/server.pp new file mode 100644 index 00000000..7e789818 --- /dev/null +++ b/types/server.pp @@ -0,0 +1,12 @@ +# lint:ignore:2sp_soft_tabs +type Proxysql::Server = Array[Hash[String, Struct[{ Optional[port] => Integer, + hostgroup_id => Integer, + Optional[status] => String[1], + Optional[weight] => Integer, + Optional[compression] => Integer, + Optional[max_connections] => Integer, + Optional[max_replication_lag] => Integer, + Optional[use_ssl] => Boolean, + Optional[max_latency_ms] => Integer, + Optional[comment] => String[1], }],1,1]] +# lint:endignore diff --git a/types/user.pp b/types/user.pp new file mode 100644 index 00000000..54424fff --- /dev/null +++ b/types/user.pp @@ -0,0 +1,13 @@ +# lint:ignore:2sp_soft_tabs +type Proxysql::User = Array[Hash[String, Struct[{ password => String[1], + default_hostgroup => Integer, + Optional[active] => Boolean, + Optional[use_ssl] => Boolean, + Optional[default_schema] => String[1], + Optional[schema_locked] => Boolean, + Optional[transaction_persistent] => Boolean, + Optional[fast_forward] => Boolean, + Optional[backend] => Boolean, + Optional[frontend] => Boolean, + Optional[max_connections] => Integer, }],1,1]] +# lint:endignore