Draft: Health master

.github/workflows/ci.yml

@@ -8,9 +8,9 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.7", "3.8"]
-        django-version: ["2.1.1", "3.1.4"]
-        database-engine: ["postgres", "mysql"]
+        python-version: [ "3.7", "3.8" ]
+        django-version: [ "2.1.1", "3.1.4" ]
+        database-engine: [ "postgres", "mysql", "mssql"]
 
     services:
       postgres:
@@ -37,6 +37,14 @@ jobs:
         ports:
           - 3306:3306
 
+
+      mssqldb:
+        image: mcr.microsoft.com/mssql/server:2017-latest
+       env:
+        ACCEPT_EULA: y
+        SA_PASSWORD: Test
+
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

binder/exceptions.py

@@ -235,3 +235,15 @@ def __add__(self, other):
 			else:
 				errors[model] = other.errors[model]
 		return BinderValidationError(errors)
+
+
+class BinderSkipSave(BinderException):
+	"""Used to abort the database transaction when validation was successfull.
+	Validation is possible when saving (post, put, multi-put) or deleting models."""
+
+	http_code = 200
+	code = 'SkipSave'
+
+	def __init__(self):
+		super().__init__()
+		self.fields['message'] = 'No validation errors were encountered.'

binder/models.py

@@ -468,8 +468,12 @@ class Meta:
 		abstract = True
 		ordering = ['pk']
 
-	def save(self, *args, **kwargs):
-		self.full_clean() # Never allow saving invalid models!
+	def save(self, *args, only_validate=False, **kwargs):
+		# A validation model might not require all validation checks as it is not a full model
+		# _validation_model can be used to skip validation checks that are meant for complete models that are actually being saved
+		self._validation_model = only_validate  # Set the model as a validation model when we only want to validate the model
+
+		self.full_clean()  # Never allow saving invalid models!
 		return super().save(*args, **kwargs)
 
 

binder/plugins/models/__init__.py

@@ -0,0 +1 @@
+from .html_field import HtmlField  # noqa: F401

binder/plugins/models/html_field.py

@@ -0,0 +1,157 @@
+from typing import List
+
+from django.db.models import TextField
+from html.parser import HTMLParser
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+
+ALLOWED_LINK_PREFIXES = [
+	'http://',
+	'https://',
+	'mailto:'
+]
+
+
+def link_rel_validator(tag, attribute_name, attribute_value) -> List[ValidationError]:
+	validation_errors = []
+
+	rels = attribute_value.split(' ')
+
+	if 'noopener' not in rels:
+
+		validation_errors.append(ValidationError(
+			_('Link needs rel="noopener"'),
+			code='invalid_attribute',
+			params={
+				'tag': tag,
+			},
+		))
+
+	if 'noreferrer' not in rels:
+		validation_errors.append(ValidationError(
+			_('Link needs rel="noreferer"'),
+			code='invalid_attribute',
+			params={
+				'tag': tag,
+			},
+		))
+
+
+	return validation_errors
+
+
+def link_validator(tag, attribute_name, attribute_value) -> List[ValidationError]:
+	validation_errors = []
+	if not any(map(lambda prefix: attribute_value.startswith(prefix), ALLOWED_LINK_PREFIXES)):
+		validation_errors.append(ValidationError(
+			_('Link is not valid'),
+			code='invalid_attribute',
+			params={
+				'tag': tag,
+			},
+		))
+	return validation_errors
+
+
+class HtmlValidator(HTMLParser):
+	allowed_tags = [
+		# General setup
+		'p', 'br',
+		# Headers
+		'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'h7',
+
+		# text decoration
+		'b', 'strong', 'i', 'em', 'u',
+		# Lists
+		'ol', 'ul', 'li',
+
+		# Special
+		'a',
+	]
+
+	allowed_attributes = {
+		'a': ['href', 'rel', 'target']
+	}
+
+	required_attributes = {
+		'a': ['rel'],
+	}
+
+	special_validators = {
+		('a', 'href'): link_validator,
+		('a', 'rel'): link_rel_validator,
+	}
+
+	error_messages = {
+		'invalid_tag': _('Tag %(tag)s is not allowed'),
+		'missing_attribute': _('Attribute %(attribute)s is required for tag %(tag)s'),
+		'invalid_attribute': _('Attribute %(attribute)s not allowed for tag %(tag)s'),
+	}
+
+	def validate(self, value: str) -> List[ValidationError]:
+		"""
+		Validates html, and gives a list of validation errors
+		"""
+
+		self.errors = []
+
+		self.feed(value)
+
+		return self.errors
+
+	def handle_starttag(self, tag: str, attrs: list) -> None:
+		tag_errors = []
+		if tag not in self.allowed_tags:
+			tag_errors.append(ValidationError(
+				self.error_messages['invalid_tag'],
+				code='invalid_tag',
+				params={
+					'tag': tag
+				},
+			))
+
+		set_attributes = set(map(lambda attr: attr[0], attrs))
+		required_attributes = set(self.required_attributes.get(tag, []))
+		missing_attributes = required_attributes - set_attributes
+		for missing_attribute in missing_attributes:
+			tag_errors.append(
+				ValidationError(
+					self.error_messages['missing_attribute'],
+					code='missing_attribute',
+					params={
+						'tag': tag,
+						'attribute': missing_attribute
+					},
+				)
+			)
+
+		allowed_attributes_for_tag = self.allowed_attributes.get(tag, [])
+
+		for (attribute_name, attribute_content) in attrs:
+			if attribute_name not in allowed_attributes_for_tag:
+				tag_errors.append(ValidationError(
+					self.error_messages['invalid_attribute'],
+					code='invalid_attribute',
+					params={
+						'tag': tag,
+						'attribute': attribute_name
+					},
+				))
+			if (tag, attribute_name) in self.special_validators:
+				tag_errors += self.special_validators[(tag, attribute_name)](tag, attribute_name, attribute_content)
+
+		self.errors += tag_errors
+
+
+class HtmlField(TextField):
+	"""
+	Determine a safe way to save "secure" user provided HTML input, and prevent XSS injections
+	"""
+
+	def validate(self, value: str, _):
+		# Validate all html tags
+		validator = HtmlValidator()
+		errors = validator.validate(value)
+
+		if errors:
+			raise ValidationError(errors)

binder/plugins/views/csvexport.py

@@ -103,7 +103,7 @@ def __init__(self, request: HttpRequest):
 		# self.writer = self.pandas.ExcelWriter(self.response)
 
 		self.work_book = self.openpyxl.Workbook()
-		self.sheet = self.work_book.create_sheet()
+		self.sheet = self.work_book._sheets[0]
 
 		# The row number we are currently writing to
 		self._row_number = 0
@@ -285,12 +285,23 @@ def get_datum(data, key, prefix=''):
 					else:
 						# Assume that we have a mapping now
 						fk_ids = data[head_key]
-						if type(fk_ids) != list:
+
+						if fk_ids is None:
+							# This case happens if we have a nullable foreign key that is null. Treat this as a many
+							# to one relation with no values.
+							fk_ids = []
+						elif type(fk_ids) != list:
 							fk_ids = [fk_ids]
 
 						# if head_key not in key_mapping:
 						prefix_key = parent_data['with_mapping'][new_prefix[1:]]
-						datums = [str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix)) for fk_id in fk_ids]
+						datums = []
+						for fk_id in fk_ids:
+							try:
+								datums.append(str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix)))
+							except KeyError:
+							 	pass
+						# datums = [str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix)) for fk_id in fk_ids]
 						return self.csv_settings.multi_value_delimiter.join(
 							datums
 						)