Initial work on #655: CSV import headers

netbox/templates/tenancy/tenant_import.html

@@ -1,40 +1,3 @@
 {% extends 'utilities/obj_import.html' %}
 
 {% block title %}Tenant Import{% endblock %}
-
-{% block instructions %}
-    <h4>CSV Format</h4>
-    <table class="table">
-        <thead>
-            <tr>
-                <th>Field</th>
-                <th>Description</th>
-                <th>Example</th>
-            </tr>
-        </thead>
-        <tbody>
-            <tr>
-                <td>Name</td>
-                <td>Tenant name</td>
-                <td>WIDG01</td>
-            </tr>
-            <tr>
-                <td>Slug</td>
-                <td>URL-friendly name</td>
-                <td>widg01</td>
-            </tr>
-            <tr>
-                <td>Group</td>
-                <td>Tenant group (optional)</td>
-                <td>Customers</td>
-            </tr>
-            <tr>
-                <td>Description</td>
-                <td>Long-form name or other text (optional)</td>
-                <td>Widgets Inc.</td>
-            </tr>
-        </tbody>
-    </table>
-    <h4>Example</h4>
-    <pre>WIDG01,widg01,Customers,Widgets Inc.</pre>
-{% endblock %}

netbox/templates/utilities/obj_import.html

@@ -28,6 +28,23 @@ <h1>{% block title %}{% endblock %}</h1>
 	</div>
 	<div class="col-md-6">
         {% block instructions %}{% endblock %}
+        {% if fields %}
+            <h4>CSV Format</h4>
+            <table class="table">
+                <tr>
+                    <th>Field</th>
+                    <th>Required</th>
+                    <th>Description</th>
+                </tr>
+                {% for name, field in fields.items %}
+                    <tr>
+                        <td><code>{{ name }}</code></td>
+                        <td>{% if field.required %}<i class="glyphicon glyphicon-ok" title="Required"></i>{% endif %}</td>
+                        <td>{{ field.help_text }}</td>
+                    </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
 	</div>
 </div>
 {% endblock %}

netbox/tenancy/forms.py

@@ -5,8 +5,7 @@
 
 from extras.forms import CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm
 from utilities.forms import (
-    APISelect, BootstrapMixin, BulkImportForm, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, CSVDataField,
-    FilterChoiceField, SlugField,
+    APISelect, BootstrapMixin, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, FilterChoiceField, SlugField,
 )
 from .models import Tenant, TenantGroup
 
@@ -36,17 +35,19 @@ class Meta:
         fields = ['name', 'slug', 'group', 'description', 'comments']
 
 
-class TenantFromCSVForm(forms.ModelForm):
-    group = forms.ModelChoiceField(TenantGroup.objects.all(), required=False, to_field_name='name',
-                                   error_messages={'invalid_choice': 'Group not found.'})
+class TenantCSVForm(forms.ModelForm):
+    group = forms.ModelChoiceField(
+        queryset=TenantGroup.objects.all(),
+        required=False,
+        to_field_name='name',
+        error_messages={
+            'invalid_choice': 'Group not found.'
+        }
+    )
 
     class Meta:
         model = Tenant
-        fields = ['name', 'slug', 'group', 'description']
-
-
-class TenantImportForm(BootstrapMixin, BulkImportForm):
-    csv = CSVDataField(csv_form=TenantFromCSVForm)
+        fields = ['name', 'slug', 'group', 'description', 'comments']
 
 
 class TenantBulkEditForm(BootstrapMixin, CustomFieldBulkEditForm):

netbox/tenancy/views.py

@@ -10,7 +10,7 @@
 from dcim.models import Site, Rack, Device
 from ipam.models import IPAddress, Prefix, VLAN, VRF
 from utilities.views import (
-    BulkDeleteView, BulkEditView, BulkImportView, ObjectDeleteView, ObjectEditView, ObjectListView,
+    BulkDeleteView, BulkEditView, BulkImportView, BulkImportView2, ObjectDeleteView, ObjectEditView, ObjectListView,
 )
 from .models import Tenant, TenantGroup
 from . import filters, forms, tables
@@ -95,9 +95,9 @@ class TenantDeleteView(PermissionRequiredMixin, ObjectDeleteView):
     default_return_url = 'tenancy:tenant_list'
 
 
-class TenantBulkImportView(PermissionRequiredMixin, BulkImportView):
+class TenantBulkImportView(PermissionRequiredMixin, BulkImportView2):
     permission_required = 'tenancy.add_tenant'
-    form = forms.TenantImportForm
+    model_form = forms.TenantCSVForm
     table = tables.TenantTable
     template_name = 'tenancy/tenant_import.html'
     default_return_url = 'tenancy:tenant_list'

netbox/utilities/forms.py

@@ -256,6 +256,60 @@ def to_python(self, value):
         return records
 
 
+class CSVDataField2(forms.CharField):
+    """
+    A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns a list of dictionaries mapping
+    column headers to values. Each dictionary represents an individual record.
+    """
+    widget = forms.Textarea
+
+    def __init__(self, fields, required_fields=[], *args, **kwargs):
+
+        self.fields = fields
+        self.required_fields = required_fields
+
+        super(CSVDataField2, self).__init__(*args, **kwargs)
+
+        self.strip = False
+        if not self.label:
+            self.label = 'CSV Data'
+        if not self.initial:
+            self.initial = ','.join(required_fields) + '\n'
+        if not self.help_text:
+            self.help_text = 'Enter one line per record. Use commas to separate values.'
+
+    def to_python(self, value):
+
+        # Python 2's csv module has problems with Unicode
+        if not isinstance(value, str):
+            value = value.encode('utf-8')
+
+        records = []
+        reader = csv.reader(value.splitlines())
+
+        # Consume and valdiate the first line of CSV data as column headers
+        headers = reader.next()
+        for f in self.required_fields:
+            if f not in headers:
+                raise forms.ValidationError('Required column header "{}" not found.'.format(f))
+        for f in headers:
+            if f not in self.fields:
+                raise forms.ValidationError('Unexpected column header "{}" found.'.format(f))
+
+        # Parse CSV data
+        for i, row in enumerate(reader, start=1):
+            if row:
+                if len(row) != len(headers):
+                    raise forms.ValidationError(
+                        "Row {}: Expected {} columns but found {}".format(i, len(headers), len(row))
+                    )
+                row = [col.strip() for col in row]
+                record = dict(zip(headers, row))
+                records.append(record)
+
+        return records
+
+
 class ExpandableNameField(forms.CharField):
     """
     A field which allows for numeric range expansion
@@ -488,7 +542,7 @@ def __init__(self, model, *args, **kwargs):
 class BulkImportForm(forms.Form):
 
     def clean(self):
-        records = self.cleaned_data.get('csv')
+        fields, records = self.cleaned_data.get('csv').split('\n', 1)
         if not records:
             return
 

netbox/utilities/views.py

@@ -6,9 +6,10 @@
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
 from django.db import transaction, IntegrityError
 from django.db.models import ProtectedError
-from django.forms import CharField, ModelMultipleChoiceField, MultipleHiddenInput, TypedChoiceField
+from django.forms import CharField, Form, ModelMultipleChoiceField, MultipleHiddenInput, TypedChoiceField
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404, redirect, render
 from django.template import TemplateSyntaxError
@@ -19,6 +20,7 @@
 from django.views.generic import View
 
 from extras.models import CustomField, CustomFieldValue, ExportTemplate, UserAction
+from utilities.forms import BootstrapMixin, CSVDataField2
 from .error_handlers import handle_protectederror
 from .forms import ConfirmationForm
 from .paginator import EnhancedPaginator
@@ -422,6 +424,85 @@ def save_obj(self, obj):
         obj.save()
 
 
+class BulkImportView2(View):
+    """
+    Import objects in bulk (CSV format).
+
+    model_form: The form used to create each imported object
+    table: The django-tables2 Table used to render the list of imported objects
+    template_name: The name of the template
+    default_return_url: The name of the URL to use for the cancel button
+    """
+    model_form = None
+    table = None
+    template_name = None
+    default_return_url = None
+
+    def _import_form(self, *args, **kwargs):
+
+        fields = self.model_form().fields.keys()
+        required_fields = [name for name, field in self.model_form().fields.items() if field.required]
+
+        class ImportForm(BootstrapMixin, Form):
+            csv = CSVDataField2(fields=fields, required_fields=required_fields)
+
+        return ImportForm(*args, **kwargs)
+
+    def get(self, request):
+
+        return render(request, self.template_name, {
+            'form': self._import_form(),
+            'fields': self.model_form().fields,
+            'return_url': self.default_return_url,
+        })
+
+    def post(self, request):
+
+        new_objs = []
+        form = self._import_form(request.POST)
+
+        if form.is_valid():
+
+            try:
+
+                # Iterate through CSV data and bind each row to a new model form instance.
+                with transaction.atomic():
+                    for row, data in enumerate(form.cleaned_data['csv'], start=1):
+                        obj_form = self.model_form(data)
+                        if obj_form.is_valid():
+                            obj = obj_form.save()
+                            new_objs.append(obj)
+                        else:
+                            for field, err in obj_form.errors.items():
+                                form.add_error('csv', "Row {} {}: {}".format(row, field, err[0]))
+                            raise ValidationError("")
+
+                # Compile a table containing the imported objects
+                obj_table = self.table(new_objs)
+
+                if new_objs:
+                    msg = 'Imported {} {}'.format(len(new_objs), new_objs[0]._meta.verbose_name_plural)
+                    messages.success(request, msg)
+                    UserAction.objects.log_import(request.user, ContentType.objects.get_for_model(new_objs[0]), msg)
+
+                    return render(request, "import_success.html", {
+                        'table': obj_table,
+                        'return_url': self.default_return_url,
+                    })
+
+            except ValidationError:
+                pass
+
+        return render(request, self.template_name, {
+            'form': form,
+            'fields': self.model_form().fields,
+            'return_url': self.default_return_url,
+        })
+
+    def save_obj(self, obj):
+        obj.save()
+
+
 class BulkEditView(View):
     """
     Edit objects in bulk.