Support adding exporter defaults from admin gui

Makefile

@@ -1,19 +1,43 @@
-test:
-	pipenv run promgen test
 .PHONY: test
-
-build:
+test: pipenv
+	pipenv run promgen test
+
+.PHONY:	all
+all: clean pipenv test build
+
+
+.PHONY: build 
+build: pipenv
 	docker-compose build
-.PHONY: build
 
+
+.PHONY:	shell
 shell:
 	docker-compose run --rm worker bash
-.PHONY:	shell
 
+
+.PHONY: docs
 docs:
 	pipenv run sphinx-build -avb html docs dist/html
-.PHONY: docs
 
-clean:
-	rm -rf .venv dist
+
+.PHONY:	pipenv
+pipenv:
+	@echo Testing if Pipenv is already installed
+	@pipenv --venv 1> /dev/null 2> /dev/null || pipenv install --dev
+
+
 .PHONY:	clean
+clean:
+	@echo Removing Pipenv
+	@pipenv --rm || true
+	@echo Clearing dist files
+	@rm -rf dist
+
+
+dump: pipenv
+	pipenv run promgen dumpdata promgen.DefaultExporter  --indent=2 --output promgen/fixtures/exporters.yaml --format=yaml
+
+load: pipenv
+	pipenv run promgen loaddata exporters
+

docker/docker-entrypoint.sh

@@ -23,7 +23,8 @@ docker-compose-bootstrap)
   done
 
   promgen migrate
-  promgen register "${PROMGEN_REGISTER_SHARD}" "${PROMGEN_REGISTER_HOST}" "${PROMGEN_REGISTER_PORT}"
+  promgen register-server "${PROMGEN_REGISTER_SHARD}" "${PROMGEN_REGISTER_HOST}" "${PROMGEN_REGISTER_PORT}"
+  promgen loaddata exporters
   exit 0
   ;;
 worker)
@@ -35,7 +36,7 @@ web)
   shift
   set -- gunicorn "promgen.wsgi:application" "$@"
   ;;
-bootstrap|createsuperuser|migrate|shell|test|import|queuecheck|rbimport|register|rules|targets|urls)
+bootstrap|createsuperuser|migrate|shell|test|import|queuecheck|rbimport|register-server|register-exporter|rules|targets|urls)
   # Shortcuts for some commonly used django commands
   set -- promgen "$@"
   ;;

docs/conf/prometheus.rst

@@ -18,8 +18,8 @@ out an updated configuration file, and update Prometheus.
 
     # Assuming we have a Prometheus shard named promshard and two servers we
     # may deploy the workers like this
-    promgen register promshard prometheus001 9090
-    promgen register promshard prometheus002 9090
+    promgen register-server promshard prometheus001 9090
+    promgen register-server promshard prometheus002 9090
 
     # Then on each Prometheus server, we would want to run a celery worker with
     # the queue name matching the name that we registered

docs/user/admin.rst

@@ -12,4 +12,4 @@ Managing Shards using CLI
 
     # Register a Prometheus server running on the host prometheus002 on port 9090
     # to the shard 'promshard'
-    promgen register promshard prometheus002 9090
+    promgen register-server promshard prometheus002 9090

promgen/admin.py

@@ -66,6 +66,12 @@ class ExporterAdmin(admin.ModelAdmin):
     readonly_fields = ('project',)
 
 
+@admin.register(models.DefaultExporter)
+class DefaultExporterAdmin(admin.ModelAdmin):
+    list_display = ('job', 'port', 'path')
+    list_filter = ('job', 'port')
+
+
 @admin.register(models.URL)
 class URLAdmin(admin.ModelAdmin):
     list_display = ('url', 'project')

promgen/context_processors.py

@@ -2,7 +2,7 @@
 # These sources are released under the terms of the MIT license: see LICENSE
 
 from django.conf import settings
-
+from promgen import models
 from promgen.version import __version__
 
 
@@ -11,4 +11,5 @@ def settings_in_view(request):
         'EXTERNAL_LINKS': settings.PROMGEN.get('links', {}),
         'TIMEZONE': settings.PROMGEN.get('timezone', 'UTC'),
         'VERSION': __version__,
+        'DEFAULT_EXPORTERS': models.DefaultExporter.objects.order_by('job', '-port'),
     }

promgen/fixtures/exporters.yaml

@@ -0,0 +1,23 @@
+# See https://github.com/prometheus/prometheus/wiki/Default-port-allocations
+# for a complete list of exporter ports. This just provides a quick add list
+# from Promgen's UI
+- model: promgen.defaultexporter
+  pk: 1
+  fields:
+    job: node
+    port: 9100
+- model: promgen.defaultexporter
+  pk: 2
+  fields:
+    job: nginx
+    port: 9113
+- model: promgen.defaultexporter
+  pk: 3
+  fields:
+    job: mysqld
+    port: 9104
+- model: promgen.defaultexporter
+  pk: 4
+  fields:
+    job: apache
+    port: 9117

promgen/management/commands/register-exporter.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2018 LINE Corporation
+# These sources are released under the terms of the MIT license: see LICENSE
+
+from django.core.management.base import BaseCommand
+
+from promgen.models import DefaultExporter
+
+
+class Command(BaseCommand):
+    help = '''Register default exporter from the commandline'''
+
+    # This is intended to be used from a configuration management tool
+    # where there may already be a port mapping that we want to import
+    # into Promgen
+
+    def add_arguments(self, parser):
+        parser.add_argument('job')
+        parser.add_argument('port', type=int)
+        parser.add_argument('path', nargs='?', default='')
+
+    def handle(self, job, port, path, **kargs):
+        exporter, created = DefaultExporter.objects.get_or_create(
+            job=job, port=port, path=path
+        )
+        if created:
+            self.stdout.write('Created {}'.format(exporter))
+        else:
+            self.stdout.write('Already exists {}'.format(exporter))
+

promgen/migrations/0006_exporter_options.py

@@ -0,0 +1,29 @@
+# Generated by Django 2.1.2 on 2018-11-28 08:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('promgen', '0005_project_owner'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DefaultExporter',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('job', models.CharField(max_length=128)),
+                ('port', models.IntegerField()),
+                ('path', models.CharField(blank=True, max_length=128)),
+            ],
+            options={
+                'ordering': ['job', 'port'],
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name='defaultexporter',
+            unique_together={('job', 'port', 'path')},
+        ),
+    ]

promgen/models.py

@@ -310,6 +310,16 @@ def get_absolute_url(self):
         return reverse('project-detail', kwargs={'pk': self.project.pk})
 
 
+class DefaultExporter(models.Model):
+    job = models.CharField(max_length=128)
+    port = models.IntegerField()
+    path = models.CharField(max_length=128, blank=True)
+
+    class Meta:
+        ordering = ['job', 'port']
+        unique_together = (('job', 'port', 'path'),)
+
+
 class URL(models.Model):
     url = models.URLField(max_length=256)
     project = models.ForeignKey('Project', on_delete=models.CASCADE)

promgen/templates/promgen/exporter_form.html

@@ -52,20 +52,20 @@ <h1>Project: {{ project.name }}</h1>
     <div class="panel panel-default">
       <div class="panel-heading">Presets</div>
       <div class="panel-body">
-{% for job, port in exporters.items|dictsort:1 %}
-      <form id="{{job}}_{{port}}" class="form-inline" action="{% url 'project-exporter' project.id %}" method="post">
+{% for default in DEFAULT_EXPORTERS %}
+      <form id="{{default.job}}_{{default.port}}" class="form-inline" action="{% url 'project-exporter' project.id %}" method="post">
           {% csrf_token %}
-          <input type="hidden" name="job" value="{{ job }}" />
-          <input type="hidden" name="port" value="{{ port }}" />
-          <input type="hidden" name="path" value="" />
+          <input type="hidden" name="job" value="{{ default.job }}" />
+          <input type="hidden" name="port" value="{{ default.port }}" />
+          <input type="hidden" name="path" value="{{ default.path }}" />
           <input type="hidden" name="enabled" value="1" />
           <div class="input-group-btn">
-            <button style="width:80%" class="btn btn-primary">Register {{ job }} (Port: {{ port }})</button>
+            <button style="width:80%" class="btn btn-primary">Register {{ default.job }} :{{ default.port }}{{ default.path }}</button>
             <a
               style="width:20%"
               class="btn btn-info promgen-exporter"
               data-href="{% url 'exporter-scrape' project.id %}"
-              data-form="#{{job}}_{{port}}"
+              data-form="#{{ default.job }}_{{ default.port }}"
               data-target="#exporterresult"
               >{% trans "Test" %}</a>
           </div>

promgen/tests/examples/promgen.yml

@@ -19,15 +19,6 @@ config_writer:
 url_writer:
   path: /etc/prometheus/blackbox.json
 
-# See https://github.com/prometheus/prometheus/wiki/Default-port-allocations
-# for a complete list of exporter ports. This just provides a quick add list
-# from Promgen's UI
-default_exporters:
-  node: 9100
-  nginx: 9113
-  mysqld: 9104
-  apache: 9117
-
 # Sender Configuration Settings
 # These match the module path of the plugin itself
 promgen.notification.email:

promgen/views.py

@@ -560,17 +560,6 @@ class ExporterRegister(LoginRequiredMixin, FormView, ProjectMixin):
     template_name = 'promgen/exporter_form.html'
     form_class = forms.ExporterForm
 
-    def get_context_data(self, **kwargs):
-        context = super(ExporterRegister, self).get_context_data(**kwargs)
-        context['exporters'] = settings.PROMGEN.get('default_exporters', {
-            'node': 9100,
-            'nginx': 9113,
-            'mysqld': 9104,
-            'apache': 9117,
-        })
-
-        return context
-
     def form_valid(self, form):
         project = get_object_or_404(models.Project, id=self.kwargs['pk'])
         exporter, _ = models.Exporter.objects.get_or_create(project=project, **form.clean())