diff --git a/README.md b/README.md index 31996af7..e209d59e 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ The disable recipe just stops redis and removes it from run levels. The cookbook also contains a recipe to allow for the installation of the redis ruby gem. -Redis-sentinel will write configuration and state data back into its configuration file. This creates obvious problems when that config is managed by chef. There is an attribute set to true which controls if chef manages the redis-sentinel config. By default chef will write out this config file and manage it. If deploying sentenel it is recommened that you set the node[:redisio][:sentinel][:manage_config] to false allowing chef to write out the initial config and then allow redis-sentiniel to manage. If running sentinel it is only advices to have node[:redisio][:sentinel][:manage_config] = true when you are pushing new changes to the config file as it will create a flapping state between chef and sentinel when sentinel writes out state to the file. +Redis-sentinel will write configuration and state data back into its configuration file. This creates obvious problems when that config is managed by chef. This cookbook will create the config file once, and then leave a breadcrumb that will guard against the file from being updated again. Recipes ------- @@ -278,11 +278,12 @@ Available options and their defaults 'ulimit' => 0 - 0 is a special value causing the ulimit to be maxconnections +32. Set to nil or false to disable setting ulimits 'configdir' => '/etc/redis' - configuration directory 'name' => nil, Allows you to name the server with something other than port. Useful if you want to use unix sockets +'tcpbacklog' => '511', 'address' => nil, Can accept a single string or an array. When using an array, the FIRST value will be used by the init script for connecting to redis 'databases' => '16', 'backuptype' => 'rdb', 'datadir' => '/var/lib/redis', -'unixoscket' => nil - The location of the unix socket to use, +'unixsocket' => nil - The location of the unix socket to use, 'unixsocketperm' => nil - The permissions of the unix socket, 'timeout' => '0', 'keepalive' => '0', @@ -293,29 +294,40 @@ Available options and their defaults 'shutdown_save' => false, 'save' => nil, # Defaults to ['900 1','300 10','60 10000'] inside of template. Needed due to lack of hash subtraction 'stopwritesonbgsaveerror' => 'yes', +'rdbcompression' => 'yes', +'rdbchecksum' => 'yes', +'dbfilename' => nil, 'slaveof' => nil, 'masterauth' => nil, 'slaveservestaledata' => 'yes', +'slavereadonly' => 'yes', 'replpingslaveperiod' => '10', 'repltimeout' => '60', +'repldisabletcpnodelay => 'no', +'slavepriority' => '100', 'requirepass' => nil, +'rename_commands' => nil, or a hash where each key is a redis command and the value is the command's new name. 'maxclients' => 10000, 'maxmemory' => nil, 'maxmemorypolicy' => nil, 'maxmemorysamples' => nil, +'appendfilename' => nil, 'appendfsync' => 'everysec', 'noappendfsynconrewrite' => 'no', 'aofrewritepercentage' => '100', 'aofrewriteminsize' => '64mb', 'luatimelimit' => '5000', 'slowloglogslowerthan' => '10000', -'slowlog-max-len' => '1024', +'slowlogmaxlen' => '1024', 'notifykeyspaceevents' => '', 'hashmaxziplistentries' => '512', 'hashmaxziplistvalue' => '64', +'listmaxziplistentries' => '512', +'listmaxziplistvalue' => '64', 'setmaxintsetentries' => '512', 'zsetmaxziplistentries' => '128', 'zsetmaxziplistvalue' => '64', +'hllsparsemaxbytes' => '3000', 'activerehasing' => 'yes', 'clientoutputbufferlimit' => [ %w(normal 0 0 0), @@ -326,7 +338,7 @@ Available options and their defaults 'aofrewriteincrementalfsync' => 'yes', 'cluster-enabled' => 'no', 'cluster-config-file' => nil, # Defaults to redis instance name inside of template if cluster is enabled. -'cluster-node-timeout' => 5, +'cluster-node-timeout' => 5000, 'includes' => nil ``` diff --git a/attributes/default.rb b/attributes/default.rb index c782a693..00d33ab4 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -53,7 +53,7 @@ default['redisio']['version'] = nil else # force version for tarball - default['redisio']['version'] = '2.8.17' + default['redisio']['version'] = '2.8.20' end # Custom installation directory @@ -77,9 +77,11 @@ 'homedir' => homedir, 'shell' => shell, 'systemuser' => true, + 'uid' => nil, 'ulimit' => 0, 'configdir' => '/etc/redis', 'name' => nil, + 'tcpbacklog' => '511', 'address' => nil, 'databases' => '16', 'backuptype' => 'rdb', @@ -95,29 +97,40 @@ 'shutdown_save' => false, 'save' => nil, # Defaults to ['900 1','300 10','60 10000'] inside of template. Needed due to lack of hash subtraction 'stopwritesonbgsaveerror' => 'yes', + 'rdbcompression' => 'yes', + 'rdbchecksum' => 'yes', + 'dbfilename' => nil, 'slaveof' => nil, 'masterauth' => nil, 'slaveservestaledata' => 'yes', + 'slavereadonly' => 'yes', 'replpingslaveperiod' => '10', 'repltimeout' => '60', + 'repldisabletcpnodelay' => 'no', + 'slavepriority' => '100', 'requirepass' => nil, + 'rename_commands' => nil, 'maxclients' => 10000, 'maxmemory' => nil, 'maxmemorypolicy' => nil, 'maxmemorysamples' => nil, + 'appendfilename' => nil, 'appendfsync' => 'everysec', 'noappendfsynconrewrite' => 'no', 'aofrewritepercentage' => '100', 'aofrewriteminsize' => '64mb', 'luatimelimit' => '5000', 'slowloglogslowerthan' => '10000', - 'slowlog-max-len' => '1024', + 'slowlogmaxlen' => '1024', 'notifykeyspaceevents' => '', 'hashmaxziplistentries' => '512', 'hashmaxziplistvalue' => '64', + 'listmaxziplistentries' => '512', + 'listmaxziplistvalue' => '64', 'setmaxintsetentries' => '512', 'zsetmaxziplistentries' => '128', 'zsetmaxziplistvalue' => '64', + 'hllsparsemaxbytes' => '3000', 'activerehasing' => 'yes', 'clientoutputbufferlimit' => [ %w(normal 0 0 0), @@ -128,7 +141,7 @@ 'aofrewriteincrementalfsync' => 'yes', 'clusterenabled' => 'no', 'clusterconfigfile' => nil, # Defaults to redis instance name inside of template if cluster is enabled. - 'clusternodetimeout' => 5, + 'clusternodetimeout' => 5000, 'includes' => nil } diff --git a/attributes/redis_sentinel.rb b/attributes/redis_sentinel.rb index 9c33a3fd..851444d0 100644 --- a/attributes/redis_sentinel.rb +++ b/attributes/redis_sentinel.rb @@ -34,6 +34,6 @@ # Manage Sentinel Config File ## Will write out the base config one time then no longer manage the config allowing sentinel to take over -default['redisio']['sentinel']['manage_config'] = true +default['redisio']['sentinel']['manage_config'] = true #deprecated default['redisio']['sentinels'] = nil diff --git a/metadata.rb b/metadata.rb index 0a6d0f65..104ef73c 100644 --- a/metadata.rb +++ b/metadata.rb @@ -15,6 +15,7 @@ recipe "redisio::install", "This recipe is used to install redis" recipe "redisio::configure", "This recipe is used to configure redis by creating the configuration files and init scripts" recipe "redisio::sentinel", "This recipe is used to configure redis sentinels by creating the configuration files and init scripts" +recipe "redisio::sentinel_enable", "This recipe is used enable sentinel init scripts" recipe "redisio::enable", "This recipe is used to start the redis instances and enable them in the default run levels" recipe "redisio::disable", "This recipe is used to stop the redis instances and disable them in the default run levels" recipe "redisio::redis_gem", "This recipe will install the redis ruby gem into the system ruby" diff --git a/providers/configure.rb b/providers/configure.rb index f7646085..f74767c1 100644 --- a/providers/configure.rb +++ b/providers/configure.rb @@ -86,18 +86,11 @@ def configure descriptors = current['ulimit'] == 0 ? current['maxclients'] + 32 : current['maxclients'] - #Manage Redisio Config? - if node['redisio']['sentinel']['manage_config'] == true - config_action = :create - else - config_action = :create_if_missing - end - recipe_eval do server_name = current['name'] || current['port'] piddir = "#{base_piddir}/#{server_name}" - aof_file = "#{current['datadir']}/appendonly-#{server_name}.aof" - rdb_file = "#{current['datadir']}/dump-#{server_name}.rdb" + aof_file = "#{current['appendfilename']}" || "#{current['datadir']}/appendonly-#{server_name}.aof" + rdb_file = "#{current['dbfilename']}" || "#{current['datadir']}/dump-#{server_name}.rdb" #Create the owner of the redis data directory user current['user'] do @@ -106,6 +99,7 @@ def configure home current['homedir'] shell current['shell'] system current['systemuser'] + uid current['uid'] unless current['uid'].nil? not_if { node['etc']['passwd']["#{current['user']}"] } end #Create the redis configuration directory @@ -191,13 +185,14 @@ def configure owner current['user'] group current['group'] mode '0644' - action config_action + action :create variables({ :version => version_hash, :piddir => piddir, :name => server_name, :job_control => node['redisio']['job_control'], :port => current['port'], + :tcpbacklog => current['tcpbacklog'], :address => current['address'], :databases => current['databases'], :backuptype => current['backuptype'], @@ -212,16 +207,24 @@ def configure :syslogfacility => current['syslogfacility'], :save => computed_save, :stopwritesonbgsaveerror => current['stopwritesonbgsaveerror'], + :rdbcompression => current['rdbcompression'], + :rdbchecksum => current['rdbchecksum'], + :dbfilename => current['dbfilename'], :slaveof => current['slaveof'], :masterauth => current['masterauth'], :slaveservestaledata => current['slaveservestaledata'], + :slavereadonly => current['slavereadonly'], :replpingslaveperiod => current['replpingslaveperiod'], :repltimeout => current['repltimeout'], + :repldisabletcpnodelay => current['repldisabletcpnodelay'], + :slavepriority => current['slavepriority'], :requirepass => current['requirepass'], + :rename_commands => current['rename_commands'], :maxclients => current['maxclients'], :maxmemory => maxmemory, :maxmemorypolicy => current['maxmemorypolicy'], :maxmemorysamples => current['maxmemorysamples'], + :appendfilename => current['appendfilename'], :appendfsync => current['appendfsync'], :noappendfsynconrewrite => current['noappendfsynconrewrite'], :aofrewritepercentage => current['aofrewritepercentage'] , @@ -232,9 +235,12 @@ def configure :notifykeyspaceevents => current['notifykeyspaceevents'], :hashmaxziplistentries => current['hashmaxziplistentries'], :hashmaxziplistvalue => current['hashmaxziplistvalue'], + :listmaxziplistentries => current['listmaxziplistentries'], + :listmaxziplistvalue => current['listmaxziplistvalue'], :setmaxintsetentries => current['setmaxintsetentries'], :zsetmaxziplistentries => current['zsetmaxziplistentries'], :zsetmaxziplistvalue => current['zsetmaxziplistvalue'], + :hllsparsemaxbytes => current['hllsparsemaxbytes'], :activerehasing => current['activerehasing'], :clientoutputbufferlimit => current['clientoutputbufferlimit'], :hz => current['hz'], @@ -244,7 +250,14 @@ def configure :clusternodetimeout => current['clusternodetimeout'], :includes => current['includes'] }) + not_if do ::File.exists?("#{current['configdir']}/#{server_name}.conf.breadcrumb") end end + + file "#{current['configdir']}/#{server_name}.conf.breadcrumb" do + content "This file prevents the chef cookbook from overwritting the redis config more than once" + action :create_if_missing + end + #Setup init.d file bin_path = node['redisio']['bin_path'] diff --git a/providers/sentinel.rb b/providers/sentinel.rb index 52ed43ca..9e7646c0 100644 --- a/providers/sentinel.rb +++ b/providers/sentinel.rb @@ -35,12 +35,6 @@ def configure #Merge the configuration defaults with the provided array of configurations provided current = current_defaults_hash.merge(current_instance_hash) - #Manage Sentinel Configs? - if node['redisio']['sentinel']['manage_config'] == true - config_action = :create - else - config_action = :create_if_missing - end recipe_eval do sentinel_name = current['name'] || current['port'] @@ -54,6 +48,8 @@ def configure home current['homedir'] shell current['shell'] system current['systemuser'] + uid current['uid'] unless current['uid'].nil? + not_if { node['etc']['passwd']["#{current['user']}"] } end #Create the redis configuration directory directory current['configdir'] do @@ -143,7 +139,7 @@ def configure owner current['user'] group current['group'] mode '0644' - action config_action + action :create variables({ :name => current['name'], :piddir => piddir, @@ -155,6 +151,12 @@ def configure :syslogfacility => current['syslogfacility'], :masters => masters_with_defaults }) + not_if do ::File.exists?("#{current['configdir']}/#{sentinel_name}.conf.breadcrumb") end + end + + file "#{current['configdir']}/#{sentinel_name}.conf.breadcrumb" do + content "This file prevents the chef cookbook from overwritting the sentinel config more than once" + action :create_if_missing end #Setup init.d file diff --git a/recipes/sentinel_enable.rb b/recipes/sentinel_enable.rb index 71b5d990..e545856a 100644 --- a/recipes/sentinel_enable.rb +++ b/recipes/sentinel_enable.rb @@ -43,7 +43,7 @@ if node['redisio']['job_control'] != 'systemd' resource.action << :enable else - link "/etc/systemd/system/multi-user.target.wants/redis@#{sentinel_name}.service" do + link "/etc/systemd/system/multi-user.target.wants/redis-sentinel@#{sentinel_name}.service" do to '/usr/lib/systemd/system/redis-sentinel@.service' notifies :run, 'execute[reload-systemd]', :immediately end diff --git a/templates/default/redis.conf.erb b/templates/default/redis.conf.erb index a8ac65e0..556d7acb 100644 --- a/templates/default/redis.conf.erb +++ b/templates/default/redis.conf.erb @@ -26,6 +26,17 @@ pidfile <%= @piddir %>/redis_<%=@name%>.pid # If port 0 is specified Redis will not listen on a TCP socket. port <%=@port%> +<% if @version[:major].to_i == 2 && @version[:minor].to_i >= 8 || @version[:major].to_i == 3 %> +# TCP listen() backlog. + +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog <%= @tcpbacklog %> +<% end %> + # If you want you can bind a single interface, if the bind option is not # specified all the interfaces will listen for incoming connections. # @@ -146,10 +157,26 @@ stop-writes-on-bgsave-error <%= @stopwritesonbgsaveerror %> # For default that's set to 'yes' as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but # the dataset will likely be bigger if you have compressible values or keys. -rdbcompression yes +rdbcompression <%= @rdbcompression %> + +<% if @version[:major].to_i == 2 && @version[:minor].to_i >= 6 || @version[:major].to_i == 3 %> +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum <%= @rdbchecksum %> +<% end %> # The filename where to dump the DB +<% if @dbfilename %> +dbfilename <%= @dbfilename %> +<% else %> dbfilename dump-<%=@name%>.rdb +<% end %> + <%end%> # The working directory. @@ -193,6 +220,24 @@ dir <%=@datadir%> # slave-serve-stale-data <%=@slaveservestaledata%> +<% if @version[:major].to_i == 2 && @version[:minor].to_i >= 6 || @version[:major].to_i == 3 %> +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only <%= @slavereadonly %> +<% end %> + # Slaves send PINGs to server in a predefined interval. It's possible to change # this interval with the repl_ping_slave_period option. The default value is 10 # seconds. @@ -208,6 +253,38 @@ repl-ping-slave-period <%=@replpingslaveperiod%> # repl-timeout <%=@repltimeout%> +<% if @version[:major].to_i == 2 && @version[:minor].to_i >= 6 || @version[:major].to_i == 3 %> +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay <%= @repldisabletcpnodelay %> +<% end %> + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one wtih priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority <%= @slavepriority %> + ################################## SECURITY ################################### # Require clients to issue AUTH before processing any other @@ -239,6 +316,12 @@ repl-timeout <%=@repltimeout%> # an empty string: # # rename-command CONFIG "" +<% if !@rename_commands.nil? %> + <% @rename_commands.each do |k, v| %> + <% v = '""' if v.empty? || v.nil? %> + <%= "rename-command #{k} #{v}" %> + <% end %> +<% end %> ################################### LIMITS #################################### @@ -333,7 +416,11 @@ appendonly yes appendonly no <%end%> # The name of the append only file (default: "appendonly.aof") +<% if @appendfilename %> +appendfilename <%= @appendfilename %> +<% else %> appendfilename appendonly-<%=@name%>.aof +<% end %> # The fsync() call tells the Operating System to actually write data on disk # instead to wait for more data in the output buffer. Some OS will really flush @@ -436,11 +523,11 @@ lua-time-limit <%= @luatimelimit %> # The following time is expressed in microseconds, so 1000000 is equivalent # to one second. Note that a negative number disables the slow log, while # a value of zero forces the logging of every command. -slowlog-log-slower-than 10000 +slowlog-log-slower-than <%= @slowloglogslowerthan %> # There is no limit to this length. Just be aware that it will consume memory. # You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 1024 +slowlog-max-len <%= @slowlogmaxlen %> <% unless @version[:major].to_i == 2 && @version[:minor].to_i >= 5 || @version[:major].to_i == 3 %> ################################ VIRTUAL MEMORY ############################### @@ -583,6 +670,12 @@ notify-keyspace-events "<%= @notifykeyspaceevents %>" hash-max-ziplist-entries <%= @hashmaxziplistentries %> hash-max-ziplist-value <%= @hashmaxziplistvalue %> +# Similarly to hashes, small lists are also encoded in a special way in order +# # to save a lot of space. The special representation is only used when +# # you are under the following limits: +list-max-ziplist-entries <%= @listmaxziplistentries %> +list-max-ziplist-value <%= @listmaxziplistvalue %> + # Sets have a special encoding in just one case: when a set is composed # of just strings that happens to be integers in radix 10 in the range # of 64 bit signed integers. @@ -596,6 +689,22 @@ set-max-intset-entries <%= @setmaxintsetentries %> zset-max-ziplist-entries <%= @zsetmaxziplistentries %> zset-max-ziplist-value <%= @zsetmaxziplistvalue %> +<% if @version[:major].to_i == 2 && @version[:minor].to_i >= 8 || @version[:major].to_i == 3 %> +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes <%= @hllsparsemaxbytes %> +<% end %> + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation redis uses (see dict.c)