Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Health master #212

Open
wants to merge 48 commits into
base: master
Choose a base branch
from
Open
Changes from 11 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
6a96bef
Fix horrible typo
Mar 29, 2021
fd3c049
Add standalone validation feature
Mar 29, 2021
ea061e6
Correctly extract querystring parameter for PUT
Mar 29, 2021
796d323
Raise the correct exception when flag is not set
Mar 29, 2021
dccd770
Improve comment
Jun 25, 2021
4280841
Fix stupid logic
Mar 29, 2021
0d33abd
Merge branch 'master' of https://github.com/CodeYellowBV/django-binde…
Jun 25, 2021
b83003c
Stop earlier when flag not set
Jun 25, 2021
0dd9ead
Merge remote-tracking branch 'jeroen/validate-only' into health_master
Jun 30, 2021
31a61bf
Add tests for validation flow.
Jun 30, 2021
9be35a2
Some cleanup
Jun 30, 2021
ba74f3c
Merge remote-tracking branch 'origin' into validate-only
Jun 30, 2021
90d72af
Merge remote-tracking branch 'origin-jeroen/validate-only' into healt…
Jul 1, 2021
32996a4
Test _validation_model field for clean
Jul 22, 2021
be297ad
Remove incorrect argument from full_clean
Jul 22, 2021
da40b8e
Merge branch 'download_csv_as_excel' into health_master
Jul 22, 2021
32b6e2d
Add tests
Jul 22, 2021
27e0ef4
Merge pull request #177 from robinzelders/whitelist_validation_errors…
Jul 22, 2021
d14787f
do not write to a new worksheet. Take the default worksheet instead
Jul 23, 2021
70622fd
update doc
Jul 23, 2021
d673457
Add first attempt at getting mssql support working
Jul 26, 2021
68397df
Merge pull request #182 from robinzelders/whitelist_validation_errors…
robinzelders Aug 4, 2021
ac8ab35
Merge branch 'master' of github.com:CodeYellowBV/django-binder
Sep 17, 2021
cfa6fa6
add support fo nullable foreign keys in CSVexport plugin
Sep 17, 2021
3fa6f5e
Merge branch 'add_support_for_nullable_fks_in_csv_export' into health…
Sep 17, 2021
26e5aa5
Merge remote-tracking branch 'origin-2/master' into health_master # C…
Dec 7, 2021
3c8b239
Do 1 query per with instead of a big combined one to prevent exponent…
daanvdk Jan 18, 2022
292d739
Do not use aggregates at all
daanvdk Jan 18, 2022
528d776
Do not use join for reverse relations
daanvdk Jan 19, 2022
3d21cbe
Merge remote-tracking branch 'origin/fix-issues-with-big-withs' into …
Jan 19, 2022
5cfc8dc
Do 1 query per with instead of a big combined one to prevent exponent…
daanvdk Jan 18, 2022
aae913f
Do not use aggregates at all
daanvdk Jan 18, 2022
034a13c
Do not use join for reverse relations
daanvdk Jan 19, 2022
22ccc40
Merge branch 'health_master' of github.com:CodeYellowBV/django-binder…
Jan 26, 2022
4db24f8
Add webpage model
Feb 18, 2022
846a54f
Implement html field
Feb 18, 2022
d47f0de
Fix flake issues
Feb 18, 2022
c4d696e
Documentation
Feb 18, 2022
4552f56
Add gettext for html field
Mar 2, 2022
06b33d7
finish sentence
Mar 2, 2022
44cab36
Add test for nested attributes & also test the content of the error m…
Mar 2, 2022
4373130
Merge multiple errors in HTMLField
Mar 3, 2022
581926f
Add noreferrer noopener check for links
Mar 3, 2022
271a279
linting
Mar 8, 2022
21288ae
Merge branch 'master' into health_master
Mar 9, 2022
2c72aff
Merge branch 'html_field' into health_master
Mar 9, 2022
15e357f
fix merge conflict
Mar 10, 2022
e96b054
Fix crash on non gotten FKs
Dec 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions binder/plugins/models/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from .html_field import HtmlField # noqa: F401
157 changes: 157 additions & 0 deletions binder/plugins/models/html_field.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
from typing import List

from django.db.models import TextField
from html.parser import HTMLParser
from django.core.exceptions import ValidationError
from django.utils.translation import gettext as _

ALLOWED_LINK_PREFIXES = [
'http://',
'https://',
'mailto:'
]


def link_rel_validator(tag, attribute_name, attribute_value) -> List[ValidationError]:
validation_errors = []

rels = attribute_value.split(' ')

if 'noopener' not in rels:

validation_errors.append(ValidationError(
_('Link needs rel="noopener"'),
code='invalid_attribute',
params={
'tag': tag,
},
))

if 'noreferrer' not in rels:
validation_errors.append(ValidationError(
_('Link needs rel="noreferer"'),
code='invalid_attribute',
params={
'tag': tag,
},
))


return validation_errors


def link_validator(tag, attribute_name, attribute_value) -> List[ValidationError]:
validation_errors = []
if not any(map(lambda prefix: attribute_value.startswith(prefix), ALLOWED_LINK_PREFIXES)):
validation_errors.append(ValidationError(
_('Link is not valid'),
code='invalid_attribute',
params={
'tag': tag,
},
))
return validation_errors


class HtmlValidator(HTMLParser):
allowed_tags = [
# General setup
'p', 'br',
# Headers
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'h7',

# text decoration
'b', 'strong', 'i', 'em', 'u',
# Lists
'ol', 'ul', 'li',

# Special
'a',
]

allowed_attributes = {
'a': ['href', 'rel', 'target']
}

required_attributes = {
'a': ['rel'],
}

special_validators = {
('a', 'href'): link_validator,
('a', 'rel'): link_rel_validator,
}

error_messages = {
'invalid_tag': _('Tag %(tag)s is not allowed'),
'missing_attribute': _('Attribute %(attribute)s is required for tag %(tag)s'),
'invalid_attribute': _('Attribute %(attribute)s not allowed for tag %(tag)s'),
}

def validate(self, value: str) -> List[ValidationError]:
"""
Validates html, and gives a list of validation errors
"""

self.errors = []

self.feed(value)

return self.errors

def handle_starttag(self, tag: str, attrs: list) -> None:
tag_errors = []
if tag not in self.allowed_tags:
tag_errors.append(ValidationError(
self.error_messages['invalid_tag'],
code='invalid_tag',
params={
'tag': tag
},
))

set_attributes = set(map(lambda attr: attr[0], attrs))
required_attributes = set(self.required_attributes.get(tag, []))
missing_attributes = required_attributes - set_attributes
for missing_attribute in missing_attributes:
tag_errors.append(
ValidationError(
self.error_messages['missing_attribute'],
code='missing_attribute',
params={
'tag': tag,
'attribute': missing_attribute
},
)
)

allowed_attributes_for_tag = self.allowed_attributes.get(tag, [])

for (attribute_name, attribute_content) in attrs:
if attribute_name not in allowed_attributes_for_tag:
tag_errors.append(ValidationError(
self.error_messages['invalid_attribute'],
code='invalid_attribute',
params={
'tag': tag,
'attribute': attribute_name
},
))
if (tag, attribute_name) in self.special_validators:
tag_errors += self.special_validators[(tag, attribute_name)](tag, attribute_name, attribute_content)

self.errors += tag_errors


class HtmlField(TextField):
"""
Determine a safe way to save "secure" user provided HTML input, and prevent XSS injections
"""

def validate(self, value: str, _):
# Validate all html tags
validator = HtmlValidator()
errors = validator.validate(value)

if errors:
raise ValidationError(errors)
5 changes: 5 additions & 0 deletions docs/plugins/html_field.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# HTML Field

The HTML field provides a django model field optimized for user posted HTML code. Its aim is to provide a safe
way to implement a CMS system, where the end user can create pages, but cannot do XSS injections.

142 changes: 142 additions & 0 deletions tests/test_html_field.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
from django.contrib.auth.models import User
from django.test import TestCase, Client

import json
from .testapp.models import Zoo, WebPage


class HtmlFieldTestCase(TestCase):

def setUp(self):
super().setUp()
u = User(username='testuser', is_active=True, is_superuser=True)
u.set_password('test')
u.save()
self.client = Client()
r = self.client.login(username='testuser', password='test')
self.assertTrue(r)

self.zoo = Zoo(name='Apenheul')
self.zoo.save()

self.webpage = WebPage.objects.create(zoo=self.zoo, content='')



def test_save_normal_text_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/', data=json.dumps({'content': 'Artis'}))
self.assertEqual(response.status_code, 200)

def test_simple_html_is_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<h1>Artis</h1><b><p>Artis is a zoo in amsterdam</a>'}))
self.assertEqual(response.status_code, 200)

def test_wrong_attribute_not_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<b onclick="">test</b>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)
self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('invalid_attribute', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])

def test_simple_link_is_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/', data=json.dumps(
{'content': '<a href="https://www.artis.nl/en/" rel="noreferrer noopener">Visit artis website</a>'}))

self.assertEqual(response.status_code, 200)



def test_javascript_link_is_not_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({
'content': '<a href="javascrt:alert(document.cookie)" rel="noreferrer noopener">Visit artis website</a>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)
self.assertEqual('ValidationError', parsed_response['code'])

self.assertEqual('invalid_attribute', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])



def test_script_is_not_ok(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<script>alert(\'hoi\');</script>'}))

self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)
self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('invalid_tag', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])

def test_script_is_not_ok_nested(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<b><script>alert(\'hoi\');</script></b>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)
self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('invalid_tag', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])


def test_can_handle_reallife_data(self):
"""
This is the worst case that we could produce on the WYIWYG edittor
"""
content = '<p>normal text</p><p><br></p><h1>HEADing 1</h1><p><br></p><h2>HEADING 2</h2><h3><br></h3><h3>HEADING 3</h3><p><br></p><p><strong>bold</strong></p><p><br></p><p><em>italic</em></p><p><br></p><p><u>underlined</u></p><p><br></p><p><a href=\"http://codeyellow.nl\" rel=\"noopener noreferrer\" target=\"_blank\">Link</a></p><p><br></p><ol><li>ol1</li><li>ol2</li></ol><ul><li>ul1</li><li>ul2</li></ul><p><br></p><p>subscripttgege</p><p>g</p>"'
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': content}))

self.assertEqual(response.status_code, 200)

def test_multiple_errors(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({
'content': '<foo><bar>Visit artis website</foo></bar>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)
self.assertEqual('ValidationError', parsed_response['code'])


self.assertEqual('invalid_tag',
parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])
self.assertEqual('invalid_tag',
parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][1]['code'])


def test_link_no_rel_errors(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<a href="https://codeyellow.nl">bla</a>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)

self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('missing_attribute',
parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])

def test_link_noopener_required(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<a href="https://codeyellow.nl" rel="noreferrer">bla</a>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)

self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('invalid_attribute',
parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])

def test_link_noreferrer_required(self):
response = self.client.put(f'/web_page/{self.webpage.id}/',
data=json.dumps({'content': '<a href="https://codeyellow.nl" rel="noopener">bla</a>'}))
self.assertEqual(response.status_code, 400)

parsed_response = json.loads(response.content)

self.assertEqual('ValidationError', parsed_response['code'])
self.assertEqual('invalid_attribute',
parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code'])
2 changes: 2 additions & 0 deletions tests/testapp/models/__init__.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@
from .zoo_employee import ZooEmployee
from .city import City, CityState, PermanentCity
from .country import Country
from .web_page import WebPage

# This is Postgres-specific
if os.environ.get('BINDER_TEST_MYSQL', '0') != '1':
from .timetable import TimeTable
13 changes: 13 additions & 0 deletions tests/testapp/models/web_page.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@

from binder.models import BinderModel
from django.db import models

from binder.plugins.models import HtmlField


class WebPage(BinderModel):
"""
Every zoo has a webpage containing some details about the zoo
"""
zoo = models.OneToOneField('Zoo', related_name='web_page', on_delete=models.CASCADE)
content = HtmlField()
1 change: 1 addition & 0 deletions tests/testapp/views/__init__.py
Original file line number Diff line number Diff line change
@@ -19,3 +19,4 @@
from .user import UserView
from .zoo import ZooView
from .zoo_employee import ZooEmployeeView
from .web_page import WebPageView
7 changes: 7 additions & 0 deletions tests/testapp/views/web_page.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from binder.views import ModelView

from ..models import WebPage

# From the api docs
class WebPageView(ModelView):
model = WebPage