387 tag support

README.md

@@ -201,6 +201,30 @@ ones set for the principal instance.
 
 So in this case you would have 3 cache instances, the first one is `cache` (you can refer to it as `cache:a` too), `cache:b` and `cache:c`. cache:a will listen on ports 2003, 2004 and 7002 for line, pickle and query respectively. But, cache:b will do it on ports 2103, 2104, and 7102, and cache:c on 2203, 2204 and 7202. All other parameters from cache:a will be inherited by cache:b and c.
 
+### Install Graphite 1.1 and enable tag-support
+
+Taged metrics are available in Graphite/Carbon 1.1.1 and later. To use tags Carbon has to add tags to a TagDB (see https://graphite.readthedocs.io/en/latest/tags.html) for details. To enable this feature in Carbon set the parameter `gr_tags_enable` to `true`. This will configure Carbon to pass tags to Graphite-web listening on `127.0.0.1` and port `gr_web_server_port`. Redis and HTTP(S) Tag DBs are not yet supported.
+
+To use tags at least version 1.1.1 of Graphite-Web/Carbopn/Whisper as well as compatible versions of Django, Django Tagging and Twisted have to be installed. 
+
+```
+class { 'graphite': 
+    gr_tags_enable            => true,
+    #
+    # Needed to install / upgrade to Graphite 1.1
+    #
+    gr_graphite_ver           => '1.1.7',
+    gr_carbon_ver             => '1.1.7',
+    gr_whisper_ver            => '1.1.7',
+    gr_django_ver             => '1.11',
+    gr_django_tagging_ver     => '0.4.6',
+    gr_twisted_ver            => '20.3.0',
+    gr_django_init_command    => 'PYTHONPATH=/opt/graphite/webapp /usr/local/bin/django-admin.py migrate --setting=graphite.settings --fake-initial',
+    gr_django_init_provider   => 'shell',
+}
+```
+
+
 ### Installing with something other than pip and specifying package names and versions
 If you need to install via something other than pip, an internal apt repo with fpm converted packages for instance, you can set `gr_pip_install` to false.
 If you're doing this you'll most likely have to override the default package names and versions as well.
@@ -757,6 +781,10 @@ Default is 10.  Time before retrying a failed remote webapp.
 
 Default is 300. Time to cache remote metric find results.
 
+##### `gr_cluster_store_merge_results`
+
+Default is 'True'. During a rebalance of a consistent hash cluster, after a partition event on a replication > 1 cluster or in other cases we might receive multiple TimeSeries data for a metric key. Merge them together rather than choosing the "most complete" one (pre-0.9.14 behaviour).  
+
 ##### `nginx_htpasswd`
 
 Default is undef (string). The user and salted SHA-1 (SSHA) password for Nginx authentication. If set, Nginx will be configured to use HTTP Basic authentication with the given user & password. e.g.: 'testuser:$jsfak3.c3Fd0i1k2kel/3sdf3'
@@ -869,6 +897,10 @@ Default is false. Set fallocate_create for whisper
 
 Default is 'False' (string). Logs timings for remote calls to carbon-cache
 
+##### `gr_log_cache_queue_sorts`
+
+Default is 'True' (String). Logs time required for the queue sorts
+
 ##### `gr_log_rendering_performance`
 
 Default is 'False' (string). Triggers the creation of rendering.log which logs timings for calls to the The Render URL API

manifests/config.pp

@@ -131,6 +131,16 @@
     seltype   => 'httpd_sys_rw_content_t',
     subscribe => Exec['Initial django db creation'],
   }
+  file { [
+    "${::graphite::graphiteweb_log_dir_REAL}/info.log",
+    "${::graphite::graphiteweb_log_dir_REAL}/exception.log"]:
+    ensure    => file,
+    group     => $gr_web_group_REAL,
+    mode      => '0755',
+    owner     => $gr_web_user_REAL,
+    seltype   => 'httpd_sys_rw_content_t',
+    subscribe => Exec['Initial django db creation'],
+  }
 
   # change access permissions for carbon-cache to align with gr_user
   # (if different from web_user)

manifests/init.pp

@@ -333,6 +333,12 @@
 #   Time before retrying a failed remote webapp.  Default = 60
 # [*gr_cluster_cache_duration*]
 #   Time to cache remote metric find results.  Default = 300
+# [*gr_cluster_store_merge_results*]
+#   During a rebalance of a consistent hash cluster, after a partition
+#   event on a replication > 1 cluster or in other cases we  might
+#   receive multiple TimeSeries data for a metric key. Merge them together
+#   rather than choosing the "most complete" one (pre-0.9.14 behaviour).
+#   Default is 'True' (String).
 # [*gr_rendering_hosts*]
 #   Array of remote rendering hosts. eg.: ['10.0.2.2:80', '10.0.2.3:80']
 #   Default is undef, setting this also enabled REMOTE_RENDER=True
@@ -423,6 +429,9 @@
 # [*gr_log_cache_performance*]
 #   logs timings for remote calls to carbon-cache
 #   Default is 'False' (String)
+# [*gr_log_cache_queue_sorts]
+#   Logs time required for the queue sorts
+#   Default is 'True' (String)
 # [*gr_log_rendering_performance*]
 #   Triggers the creation of rendering.log which logs timings for calls to
 #   the The Render URL API
@@ -446,6 +455,9 @@
 # [*gr_django_init_command]
 #   Command to use for the Django DB initialization exec.
 #   default: "${::graphite::params::python_binary} manage.py syncdb --noinput"
+#   Since Django 1.7 sysndb is deprecated and the command should be
+#   "${::graphite::params::python_binary} manage.py migrate --run-syncd" or
+#   'django-admin migrate --settings=graphite.settings --run-syncdb'
 # [*gr_django_tagging_pkg*]
 #   String. The name of the django tagging package to install
 #   Default: django-tagging
@@ -527,6 +539,9 @@
 #   Default: false
 # [*gr_enable_logrotation*]
 #   Boolean. Sets up a cronjob to rotate carbon and webapp logs.
+# [*gr_tags_enable*]
+#   Boolean. Enable tag support in carbon. Tags will be sent to the graphite-web service
+#   on 'http://127.0.0.1:${gr_web_server_port}'
 # [*gr_apache_port*]
 #   DEPRECATED. Use `gr_web_server_port` now. Trying to set this variable will
 #   cause puppet to fail.
@@ -697,6 +712,7 @@
   $gr_cluster_find_timeout                = 2.5,
   $gr_cluster_retry_delay                 = 60,
   $gr_cluster_cache_duration              = 300,
+  $gr_cluster_store_merge_results         = 'True',
   $nginx_htpasswd                         = undef,
   $nginx_proxy_read_timeout               = 10,
   $manage_ca_certificate                  = true,
@@ -736,6 +752,7 @@
   $gr_whisper_lock_writes                 = 'False',
   $gr_whisper_fallocate_create            = 'False',
   $gr_log_cache_performance               = 'False',
+  $gr_log_cache_queue_sorts               = 'False',
   $gr_log_rendering_performance           = 'False',
   $gr_log_metric_access                   = 'False',
   $wsgi_processes                         = 5,
@@ -778,7 +795,8 @@
   $gr_rendering_hosts_timeout             = '1.0',
   $gr_prefetch_cache                      = undef,
   $gr_apache_port                         = undef,
-  $gr_apache_port_https                   = undef,) inherits graphite::params {
+  $gr_apache_port_https                   = undef,
+  $gr_tags_enable                         = false,) inherits graphite::params {
   # Validation of input variables.
   # TODO - validate all the things
   validate_string($gr_use_remote_user_auth)
@@ -796,6 +814,7 @@
   validate_bool($gr_manage_python_packages)
   validate_bool($gr_disable_webapp_cache)
   validate_bool($gr_base_dir_managed_externally)
+  validate_bool($gr_tags_enable)
 
   if $gr_apache_port or $gr_apache_port_https {
     fail('$gr_apache_port and $gr_apache_port_https are deprecated in favour of $gr_web_server_port and $gr_web_server_port_https')

manifests/install.pp

@@ -127,23 +127,34 @@
     }
 
     # hack unusual graphite install target
-    $carbon = "\"carbon-\"*\"-py${::graphite::params::pyver}.egg-info\""
-    $gweb = "\"graphite_web-\"*\"-py${::graphite::params::pyver}.egg-info\""
-    create_resources('exec', {
-      'carbon_hack' => {
-        command => "ln -s \"${::graphite::base_dir_REAL}/lib/\"${carbon} \"${::graphite::params::libpath}/\""
-      }
-      ,
-      'gweb_hack'   => {
-        command => "ln -s \"${::graphite::base_dir_REAL}/webapp/\"${gweb} \"${::graphite::params::libpath}/\""
-      }
-      ,
+    $carbon = "carbon-${::graphite::gr_carbon_ver}-py${::graphite::params::pyver}.egg-info"
+    $gweb = "graphite_web-${::graphite::gr_graphite_ver}-py${::graphite::params::pyver}.egg-info"
+    exec{ 'gweb_hack':
+        command   => "ln -s '${::graphite::base_dir_REAL}/webapp/${gweb}' '${::graphite::params::libpath}/'",
+        unless    => "test -L '${::graphite::params::libpath}/${gweb}'",
+        provider  => 'shell',
+        subscribe => Package['graphite-web']
     }
-    , {
-      refreshonly => true,
-      subscribe   => Package['carbon', 'graphite-web', 'whisper'],
-      provider    => 'shell',
+    exec{ 'carbon_hack':
+        command   => "ln -s '${::graphite::base_dir_REAL}/lib/${carbon}' '${::graphite::params::libpath}/'",
+        unless    => "test -L '${::graphite::params::libpath}/${carbon}'",
+        provider  => 'shell',
+        subscribe => Package['carbon']
+    }
+    # Purge duplicate egg-info files having the wrong version
+    exec{ 'Purge old graphite-web egg-info':
+        command   => "find '${::graphite::base_dir_REAL}/webapp' '${::graphite::params::libpath}' -iname 'graphite_web-*.egg-info' -not  -iname '${gweb}' | xargs rm -rf",
+        onlyif    => "ls -d \"${::graphite::base_dir_REAL}/webapp/\"graphite_web*.egg-info \"${::graphite::params::libpath}/\"graphite_web*.egg-info | grep -v '${gweb}'",
+        path      => '/bin:/usr/bin',
+        provider  => 'shell',
+        subscribe => Package['graphite-web']
+    }
+    exec{ 'Purge old carbon egg-info':
+        command   => "find '${::graphite::base_dir_REAL}/lib' '${::graphite::params::libpath}' -iname 'carbon-*.egg-info' -not  -iname '${carbon}' | xargs rm -rf",
+        onlyif    => "ls -d \"${::graphite::base_dir_REAL}/lib/\"carbon*.egg-info \"${::graphite::params::libpath}/\"carbon*.egg-info | grep -v '${carbon}'",
+        path      => '/bin:/usr/bin',
+        provider  => 'shell',
+        subscribe => Package['carbon']
     }
-    )
   }
 }

spec/acceptance/nodesets/docker.yml

@@ -0,0 +1,25 @@
+HOSTS:
+  graphite-0.9:
+    roles:
+      - default
+      - graphite_1
+    platform: ubuntu-1804-x86_64
+    image: ubuntu:bionic
+    hypervisor: docker
+    docker_cmd: '["/sbin/init"]'
+    docker_image_commands:
+    - 'apt install -y init cron iproute2 libffi-dev netcat-openbsd'
+
+  graphite-1.1-fresh:
+    roles:
+      - graphite_1
+    platform: ubuntu-1804-x86_64
+    image: ubuntu:bionic
+    hypervisor: docker
+    docker_cmd: '["/sbin/init"]'
+    docker_image_commands:
+    - 'apt install -y init cron iproute2 libffi-dev netcat-openbsd'
+
+
+CONFIG:
+    type: foss

spec/acceptance/x_gr_1_inst_up_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper_acceptance'
+
+hosts_as('graphite_1').each do |graphite_host|
+
+  describe "install/update to Graphite 1.1 on host #{graphite_host}" do
+    let(:graphite_version){ '1.1.7' }
+    let(:django_version){ '1.11' }
+    let(:django_tagging_version){ '0.4.6' }
+    let(:twisted_version){ '20.3.0' }
+
+    it 'has to delete the existing django db to work around failures migrating the database from 0.9' do
+      on(graphite_host, 'rm -f /opt/graphite/storage/graphite.db || exit 0')
+    end
+
+    context 'install or upgrade Graphite' do
+      # Using puppet_apply as a helper
+      it 'should apply with no errors' do
+        pp = <<-EOS
+          class { 'graphite': 
+                secret_key                => '123456789', 
+                gr_base_dir               => '/opt/graphite',
+                gr_django_init_command    => 'PYTHONPATH=/opt/graphite/webapp /usr/local/bin/django-admin.py migrate --setting=graphite.settings --fake-initial',
+                gr_django_init_provider   => 'shell',
+                gr_carbon_ver             => '#{graphite_version}',
+                gr_graphite_ver           => '#{graphite_version}',
+                gr_whisper_ver            => '#{graphite_version}',
+                gr_django_ver             => '#{django_version}',
+                gr_django_tagging_ver     => '#{django_tagging_version}',
+                gr_twisted_ver            => '#{twisted_version}',
+          }
+        EOS
+
+        apply_manifest_on(graphite_host, pp, :catch_failures => true)
+      end
+
+      it 'should send metric data to carbon' do
+        result = on(graphite_host, 'echo "one.metric 42 `date +%s`" | nc -N 127.0.0.1 2003')
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'should get data from graphite' do
+        result = on(graphite_host, "sleep 10 && curl -s 'http://127.0.0.1/render?target=one.metric&format=raw&from=-5min' | grep 'one.metric,.*42.0'")
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'exist only one link to the egg-info of graphite in the python lib directory' do
+        result = on(graphite_host, "test `ls -d /usr/local/lib/python2.7/dist-packages/graphite_web*.egg-info | wc -l` -eq 1")
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'exist only one egg-info of graphite in the installation directory' do
+        result = on(graphite_host, "test `ls -d /opt/graphite/webapp/graphite_web*.egg-info | wc -l` -eq 1")
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'exist only one link to the egg-info of cabon in the python lib directory' do
+        result = on(graphite_host, "test `ls -d /usr/local/lib/python2.7/dist-packages/carbon*.egg-info | wc -l` -eq 1")
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'exist only one egg-info of carbon in the installation directory' do
+        result = on(graphite_host, "test `ls -d /opt/graphite/lib/carbon*.egg-info | wc -l` -eq 1")
+        expect(result.exit_code).to eq 0
+      end
+    end
+  end
+
+end

spec/acceptance/x_tag_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper_acceptance'
+
+hosts_as('graphite_1').each do |graphite_host|
+
+  describe "graphite 1.1 install/update on host #{graphite_host}" do
+    let(:graphite_version){ '1.1.7' }
+    let(:django_version){ '1.11' }
+    let(:django_tagging_version){ '0.4.6' }
+    let(:twisted_version){ '20.3.0' }
+
+    it 'has to delete the existing django db to work around failures migrating the database from 0.9' do
+      on(graphite_host, 'rm -f /opt/graphite/storage/graphite.db || exit 0')
+    end
+
+    context 'install or upgrade Graphite 1.1 with tag support' do
+      # Using puppet_apply as a helper
+      it 'should work with no errors' do
+        pp = <<-EOS
+          class { 'graphite': 
+                secret_key                => '123456789', 
+                gr_base_dir               => '/opt/graphite',
+                gr_django_init_command    => 'PYTHONPATH=/opt/graphite/webapp /usr/local/bin/django-admin.py migrate --setting=graphite.settings --fake-initial && chown www-data /opt/graphite/storage/log/*.log',
+                gr_django_init_provider   => 'shell',
+                gr_carbon_ver             => '#{graphite_version}',
+                gr_graphite_ver           => '#{graphite_version}',
+                gr_whisper_ver            => '#{graphite_version}',
+                gr_django_ver             => '#{django_version}',
+                gr_django_tagging_ver     => '#{django_tagging_version}',
+                gr_twisted_ver            => '#{twisted_version}',
+                gr_tags_enable            => true,
+          }
+        EOS
+
+        apply_manifest_on(graphite_host, pp, :catch_failures => true)
+      end
+
+      it 'should send tagged data' do
+        result = on(graphite_host, 'echo "data;arg=key 42 `date +%s`" | nc -N 127.0.0.1 2003')
+        expect(result.exit_code).to eq 0
+      end
+
+      it 'graphite should have received the data' do
+        result = on(graphite_host, "sleep 10 && curl -s 'http://127.0.0.1/render?target=seriesByTag(\"arg=key\")&format=raw&from=-5min' | grep 'arg=key,.*42.0'")
+        expect(result.exit_code).to eq 0
+      end
+    end
+  end
+end
+