Scheduled monthly dependency update for January (#148)

* Update sqlalchemy from 1.1.15 to 1.2.0

* Update aiohttp from 2.3.5 to 2.3.7

* Update raven from 6.3.0 to 6.4.0

* Update pytest from 3.3.0 to 3.3.1

* Update pytest-aiohttp from 0.2.0 to 0.3.0

* rollback SQLAlchemy, ref aio-libs/aiopg#412

* tweaks and fixes while deploying to beta

README.md

@@ -8,6 +8,6 @@ Backend application for [TutorCruncher's](https://tutorcruncher.com) web integra
 
 # LICENSE
 
-Copyright TutorCruncher ltd. 2017.
+Copyright TutorCruncher ltd. 2017,2018.
 
 All rights reserved.

tcsocket/app/index.html

@@ -38,13 +38,15 @@
                    \____/ \___/ \___|_|\_\___|\__|
 </pre>
 <section>
-  COMMIT: {{ commit }}
+  COMMIT: {{ COMMIT }}
   <br>
-  RELEASE DATE: {{ release_date }}
+  RELEASE_DATE: {{ RELEASE_DATE }}
+  <br>
+  SERVER_NAME: {{ SERVER_NAME }}
 </section>
 <section>
-  You're looking at TutorCruncher socket's API, for more
-  information see <a href="https://tutorcruncher.com/socket">tutorcruncher.com/socket</a>.
+  You're looking at TutorCruncher socket's API, for more information see
+  <a href="https://tutorcruncher.com/features/tutorcruncher-socket/">tutorcruncher.com/features/tutorcruncher-socket/</a>.
 </section>
 <section>
   For help integrating with socket, see

tcsocket/app/main.py

@@ -49,8 +49,9 @@ def create_app(loop, *, settings: Settings=None):
     app['settings'] = settings
 
     ctx = dict(
-        commit=os.getenv('COMMIT', '-'),
-        release_date=os.getenv('RELEASE_DATE', '-'),
+        COMMIT=os.getenv('COMMIT', '-'),
+        RELEASE_DATE=os.getenv('RELEASE_DATE', '-'),
+        SERVER_NAME=os.getenv('SERVER_NAME', '-'),
     )
     index_html = (THIS_DIR / 'index.html').read_text()
     for key, value in ctx.items():

tcsocket/app/validation.py

@@ -2,7 +2,7 @@
 from enum import Enum, unique
 from typing import Any, List, Optional
 
-from pydantic import BaseModel, EmailStr, NoneStr, constr
+from pydantic import BaseModel, EmailStr, NoneStr, constr, validator
 
 EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date'
 
@@ -89,9 +89,13 @@ class EnquiryModal(BaseModel):
     # TODO:
     # subject: Optional[int] = None
     # qual_level: Optional[int] = None
-    upstream_http_referrer: Optional[constr(max_length=1023)] = None
+    upstream_http_referrer: Optional[str] = None
     grecaptcha_response: constr(min_length=20, max_length=1000)
 
+    @validator('upstream_http_referrer')
+    def val_upstream_http_referrer(cls, v):
+        return v[:1023]
+
 
 VIEW_MODELS = {
     'company-create': CompanyCreateModal,

tcsocket/app/views.py

@@ -387,11 +387,13 @@ async def enquiry(request):
     company = dict(request['company'])
     if request.method == METH_POST:
         data = request['json_obj']
+        data = {k: v for k, v in data.items() if v is not None}
         x_forward_for = request.headers.get('X-Forwarded-For')
+        referrer = request.headers.get('Referer')
         data.update(
             user_agent=request.headers.get('User-Agent'),
             ip_address=x_forward_for and x_forward_for.split(',', 1)[0].strip(' '),
-            http_referrer=request.headers.get('Referer'),
+            http_referrer=referrer and referrer[:1023],
         )
         await request.app['worker'].submit_enquiry(company, data)
         return public_json_response(request, status='enquiry submitted to TutorCruncher', status_=201)

tcsocket/requirements.txt

@@ -1,12 +1,12 @@
 SQLAlchemy==1.1.15
 aiodns==1.1.1
-aiohttp==2.3.5
+aiohttp==2.3.7
 aiopg==0.13.1
 arq==0.13
 cchardet==2.1.1
 gunicorn==19.7.1
 python-dateutil==2.6.1
 pillow==4.3.0
 pydantic==0.6.3
-raven==6.3.0
+raven==6.4.0
 uvloop==0.9.1

tests/end2end.sh

@@ -18,7 +18,7 @@ fi
 echo "allow all;" > nginx/prod/allowed.nginx.conf
 export LOGSPOUT_ENDPOINT="syslog://example.com"
 export APP_MASTER_KEY="123"
-export SERVER_NAME="scaleway"
+export SERVER_NAME="end2endtest"
 export MODE="PRODUCTION"
 export RAVEN_DSN="-"
 

tests/requirements.txt

@@ -2,8 +2,8 @@ coverage==4.4.2
 flake8==3.5.0
 pycodestyle==2.3.1
 pyflakes==1.6.0
-pytest==3.3.0
-pytest-aiohttp==0.2.0
+pytest==3.3.1
+pytest-aiohttp==0.3.0
 pytest-cov==2.5.1
 pytest-isort==0.1.0
 pytest-mock==1.6.3

tests/test_public_views.py

@@ -231,12 +231,7 @@ async def test_post_enquiry(cli, company, other_server):
             'enquiry_post',
             {
                 'client_name': 'Cat Flap',
-                'client_email': None,
                 'client_phone': '123',
-                'service_recipient_name': None,
-                'attributes': None,
-                'contractor': None,
-                'upstream_http_referrer': None,
                 'user_agent': 'Testing Browser',
                 'ip_address': None,
                 'http_referrer': None,
@@ -309,12 +304,7 @@ async def test_post_enquiry_400(cli, company, other_server, caplog):
             'enquiry_post',
             {
                 'client_name': 'Cat Flap',
-                'client_email': None,
                 'client_phone': '123',
-                'service_recipient_name': None,
-                'attributes': None,
-                'contractor': None,
-                'upstream_http_referrer': None,
                 'user_agent': 'Testing Browser',
                 'ip_address': None,
                 'http_referrer': 'http://cause400.com',
@@ -341,11 +331,6 @@ async def test_post_enquiry_skip_grecaptcha(cli, company, other_server):
             'enquiry_post',
             {
                 'client_name': 'Cat Flap',
-                'client_email': None,
-                'client_phone': None,
-                'service_recipient_name': None,
-                'attributes': None,
-                'contractor': None,
                 'upstream_http_referrer': 'foobar',
                 'user_agent': 'Testing Browser',
                 'ip_address': None,
@@ -365,6 +350,22 @@ async def test_post_enquiry_500(cli, company, other_server, caplog):
     assert 'Bad response from http://localhost:' in caplog
 
 
+async def test_post_enquiry_referrer_too_long(cli, company, other_server):
+    data = {
+        'client_name': 'Cat Flap',
+        'client_phone': '123',
+        'grecaptcha_response': 'good' * 5,
+        'upstream_http_referrer': 'X' * 2000
+    }
+    url = cli.server.app.router['enquiry'].url_for(company=company.public_key)
+    r = await cli.post(url, data=json.dumps(data), headers={'User-Agent': 'Testing Browser', 'Referer': 'Y' * 2000})
+    assert r.status == 201, await r.text()
+    data = await r.json()
+    assert data == {'status': 'enquiry submitted to TutorCruncher'}
+    assert other_server.app['request_log'][1][1]['upstream_http_referrer'] == 'X' * 1023
+    assert other_server.app['request_log'][1][1]['http_referrer'] == 'Y' * 1023
+
+
 async def snap(request):
     raise RuntimeError('snap')