From ed107eecedbb42a43b54e5eda64b4d72cdec6f08 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 9 Aug 2021 11:10:36 -0700 Subject: [PATCH 01/38] add jinja to setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 034a4704..e1617d09 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ def read(fname): description="Reference Matrix Identity Verification and Lookup Server", python_requires=">=3.6", install_requires=[ + "jinja2", "signedjson==1.1.1", "unpaddedbase64==1.1.0", "Twisted>=18.4.0", From 338bd5dd1202dc815333566a49f85886b870b643 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 9 Aug 2021 11:11:33 -0700 Subject: [PATCH 02/38] update send_email function to render templates with jinja --- sydent/util/emailutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index e9de0ac6..597b333a 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -22,6 +22,8 @@ from html import escape from typing import TYPE_CHECKING, Dict +from jinja2 import Template + import twisted.python.log from sydent.util import time_msec @@ -73,7 +75,10 @@ def sendEmail( # conflicting with it. allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) - mailString = open(templateFile).read() % allSubstitutions + with open(templateFile) as file_: + template = Template(file_.read()) + mailString = template.render(allSubstitutions) + parsedFrom = email.utils.parseaddr(mailFrom)[1] parsedTo = email.utils.parseaddr(mailTo)[1] if parsedFrom == "" or parsedTo == "": From 589c396b1a5efc9d7cdf73d502f4e9ea2487bb33 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 9 Aug 2021 11:13:04 -0700 Subject: [PATCH 03/38] change templates to use jinja --- .../res/is-test/invite_template.eml | 12 +++--- .../res/is-test/verification_template.eml | 2 +- res/matrix-org/invite_template.eml | 38 ++++++++++--------- res/matrix-org/verification_template.eml | 26 ++++++------- res/vector-im/invite_template.eml | 30 +++++++-------- res/vector-im/verification_template.eml | 20 +++++----- sydent/terms/terms.py | 2 +- 7 files changed, 66 insertions(+), 64 deletions(-) diff --git a/matrix_is_test/res/is-test/invite_template.eml b/matrix_is_test/res/is-test/invite_template.eml index 557a88cb..0b09cb12 100644 --- a/matrix_is_test/res/is-test/invite_template.eml +++ b/matrix_is_test/res/is-test/invite_template.eml @@ -1,8 +1,8 @@ { - "token": "%(token)s", - "room_alias": "%(room_alias)s", - "room_avatar_url": "%(room_avatar_url)s", - "room_name": "%(room_name)s", - "sender_display_name": "%(sender_display_name)s", - "sender_avatar_url": "%(sender_avatar_url)s" + "token": "{{ token }}", + "room_alias": "{{ room_alias }}", + "room_avatar_url": "{{ room_avatar_url }}", + "room_name": "{{ room_name }}", + "sender_display_name": "{{ sender_display_name }}", + "sender_avatar_url": "{{ sender_avatar_url }}" } diff --git a/matrix_is_test/res/is-test/verification_template.eml b/matrix_is_test/res/is-test/verification_template.eml index c75dd7dc..d74f9a83 100644 --- a/matrix_is_test/res/is-test/verification_template.eml +++ b/matrix_is_test/res/is-test/verification_template.eml @@ -1 +1 @@ -<<<%(token)s>>> +<<<{{ token }}>>> diff --git a/res/matrix-org/invite_template.eml b/res/matrix-org/invite_template.eml index dfcd7216..f1205651 100644 --- a/res/matrix-org/invite_template.eml +++ b/res/matrix-org/invite_template.eml @@ -1,25 +1,25 @@ -Date: %(date)s -From: %(from)s -To: %(to)s -Message-ID: %(messageid)s -Subject: %(subject_header_value)s +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} +Subject: {{ subject_header_value }} MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="%(multipart_boundary)s" + boundary="{{ multipart_boundary }}" ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -%(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room -%(bracketed_room_name)son Matrix. To join the conversation, either pick a +{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room +{{ bracketed_room_name }}on Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use -the single-click link below to join via Element (requires Chrome, Firefox, -Safari, iOS or Android) +the single-click link below to join via Element requires Chrome, Firefox, +Safari, iOS or Android -%(web_client_location)/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s +{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} About Matrix: @@ -39,7 +39,7 @@ Thanks, Matrix ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -83,6 +83,8 @@ pre, code { text-align: right; margin-left: 20px; } + + @@ -102,18 +104,18 @@ pre, code {

Hi,

-

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a room %(bracketed_room_name_forhtml)s on +

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a room {{ bracketed_room_name_forhtml }} on Matrix. To join the conversation, either pick a Matrix client or use the single-click -link below to join via Element (requires +link below to join via Element requires Chrome, Firefox or Safari on the web, -or iOS or Android on mobile.)

+or iOS or Android on mobile.

Join the conversation. + href="https://app.element.io/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }}">Join the conversation.


@@ -140,4 +142,4 @@ create new communication solutions or extend the capabilities and reach of exist ---%(multipart_boundary)s-- +--{{ multipart_boundary }}-- diff --git a/res/matrix-org/verification_template.eml b/res/matrix-org/verification_template.eml index d98e4d9e..a9c7261e 100644 --- a/res/matrix-org/verification_template.eml +++ b/res/matrix-org/verification_template.eml @@ -1,13 +1,13 @@ -Date: %(date)s -From: %(from)s -To: %(to)s -Message-ID: %(messageid)s +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} Subject: Confirm your email address for Matrix MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="%(multipart_boundary)s" + boundary="{{ multipart_boundary }}" ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -17,9 +17,9 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -%(link)s +{{ link }} -If your client requires a code, the code is %(token)s +If your client requires a code, the code is {{ token }} If you aren't aware of making such a request, please disregard this email. @@ -35,7 +35,7 @@ Matrix defines the standard, and provides open source reference implementations Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -59,13 +59,13 @@ body { identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

...or copy this link into your web browser:

-

%(link)s

+

{{ link }}

-

If your client requires a code, the code is %(token)s

+

If your client requires a code, the code is {{ token }}

If you aren't aware of making such a request, please disregard this email.

@@ -85,4 +85,4 @@ create new communication solutions or extend the capabilities and reach of exist ---%(multipart_boundary)s-- +--{{ multipart_boundary }}-- diff --git a/res/vector-im/invite_template.eml b/res/vector-im/invite_template.eml index 6ee601f6..991b8f15 100644 --- a/res/vector-im/invite_template.eml +++ b/res/vector-im/invite_template.eml @@ -1,23 +1,23 @@ -Date: %(date)s -From: %(from)s -To: %(to)s -Message-ID: %(messageid)s -Subject: %(subject_header_value)s +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} +Subject: {{ subject_header_value }} MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="%(multipart_boundary)s" + boundary="{{ multipart_boundary }}" ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -%(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room -%(bracketed_room_name)son Element. To join the conversation please follow the +{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room +{{ bracketed_room_name }}on Element. To join the conversation please follow the link below. -%(web_client_location)/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s +{{ web_client_location)/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, @@ -52,7 +52,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -123,12 +123,12 @@ pre, code {

Hi,

-

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a -room %(bracketed_room_name_forhtml)s on Element.

+

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a +room {{ bracketed_room_name_forhtml }} on Element.

Join the conversation. + href="https://app.element.io/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }}">Join the conversation.

Element is an open source collaboration app built on the Matrix.org @@ -174,4 +174,4 @@ Matrix visit https://matrix.org.

---%(multipart_boundary)s-- +--{{ multipart_boundary }}-- diff --git a/res/vector-im/verification_template.eml b/res/vector-im/verification_template.eml index e13b7e2c..01e592f7 100644 --- a/res/vector-im/verification_template.eml +++ b/res/vector-im/verification_template.eml @@ -1,13 +1,13 @@ -Date: %(date)s -From: %(from)s -To: %(to)s -Message-ID: %(messageid)s +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} Subject: Confirm your email address for Element MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="%(multipart_boundary)s" + boundary="{{ multipart_boundary }}" ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -19,7 +19,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -%(link)s +{{ link }} Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. @@ -51,7 +51,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---%(multipart_boundary)s +--{{ multipart_boundary }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -125,7 +125,7 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

Please note that Element requires Chrome, @@ -167,4 +167,4 @@ pre, code { ---%(multipart_boundary)s-- +--{{ multipart_boundary }}-- diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index 58bf83f1..dc16108c 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.full_load(fp) + termsYaml = yaml.safe_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: From 8e40335e67dedd3212f392f7462ad3399b4dcc39 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 9 Aug 2021 11:14:23 -0700 Subject: [PATCH 04/38] lints --- sydent/util/emailutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 597b333a..9ac9a06e 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -22,9 +22,8 @@ from html import escape from typing import TYPE_CHECKING, Dict -from jinja2 import Template - import twisted.python.log +from jinja2 import Template from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength From 7c81e2230c1f584f0ae39a312406451fc105af3b Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 9 Aug 2021 11:17:00 -0700 Subject: [PATCH 05/38] Add changelog Signed-off-by: H.Shay --- changelog.d/376.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/376.misc diff --git a/changelog.d/376.misc b/changelog.d/376.misc new file mode 100644 index 00000000..a7d5f34a --- /dev/null +++ b/changelog.d/376.misc @@ -0,0 +1 @@ +Use jinja for templating emails. \ No newline at end of file From a2106c11a99714751dcc10762c1c82b2aa451625 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 08:32:06 -0700 Subject: [PATCH 06/38] add .j2 templates --- .../res/is-test/invite_template.eml.j2 | 8 + .../res/is-test/verification_template.eml.j2 | 1 + res/vector-im/invite_template.eml.j2 | 177 ++++++++++++++++++ res/vector-im/verification_template.eml.j2 | 170 +++++++++++++++++ 4 files changed, 356 insertions(+) create mode 100644 matrix_is_test/res/is-test/invite_template.eml.j2 create mode 100644 matrix_is_test/res/is-test/verification_template.eml.j2 create mode 100644 res/vector-im/invite_template.eml.j2 create mode 100644 res/vector-im/verification_template.eml.j2 diff --git a/matrix_is_test/res/is-test/invite_template.eml.j2 b/matrix_is_test/res/is-test/invite_template.eml.j2 new file mode 100644 index 00000000..0b09cb12 --- /dev/null +++ b/matrix_is_test/res/is-test/invite_template.eml.j2 @@ -0,0 +1,8 @@ +{ + "token": "{{ token }}", + "room_alias": "{{ room_alias }}", + "room_avatar_url": "{{ room_avatar_url }}", + "room_name": "{{ room_name }}", + "sender_display_name": "{{ sender_display_name }}", + "sender_avatar_url": "{{ sender_avatar_url }}" +} diff --git a/matrix_is_test/res/is-test/verification_template.eml.j2 b/matrix_is_test/res/is-test/verification_template.eml.j2 new file mode 100644 index 00000000..d74f9a83 --- /dev/null +++ b/matrix_is_test/res/is-test/verification_template.eml.j2 @@ -0,0 +1 @@ +<<<{{ token }}>>> diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 new file mode 100644 index 00000000..991b8f15 --- /dev/null +++ b/res/vector-im/invite_template.eml.j2 @@ -0,0 +1,177 @@ +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} +Subject: {{ subject_header_value }} +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="{{ multipart_boundary }}" + +--{{ multipart_boundary }} +Content-Type: text/plain; charset=UTF-8 +Content-Disposition: inline + +Hi, + +{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room +{{ bracketed_room_name }}on Element. To join the conversation please follow the +link below. + +{{ web_client_location)/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} + +Element is an open source collaboration app built on the Matrix.org +open standard for interoperable communication: supporting group chat, +file transfer, voice and video calling, integrations to other apps, bridges +to other communication systems and much more. + +Please note that you will need to use Chrome, Firefox or Safari on the web, or +iOS or Android on mobile. + +Thanks, + +Element + + +About Element: + +Break through - Element allows teams to communicate across a wide range of collaboration +apps. If some team members use Element while others use IRC, Slack or Gitter, Element will +allow these team members to seamlessly work together. Element offers the richest +network of communication bridges. + +Own Your Own Data - No one should control your communication and data but you. Element +lets you run your own server, and provides users and teams with the most advanced +crypto ratchet technology available today for a decentralized secure Internet. + +Open Source - Element is entirely open source: all the code is published on GitHub +(Apache License) for anyone to see and extend. This means teams can customize or +contribute to the code and everyone can benefit from the speed of community innovation. + +Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, +decentralized communication delivering a community of users, bridged networks, +integrated bots and applications plus full end-to-end encryption. To learn more about +Matrix visit https://matrix.org. + +--{{ multipart_boundary }} +Content-Type: text/html; charset=UTF-8 +Content-Disposition: inline + + + + + + + + + + + + + +
+ + + + + +
+
+ +

Hi,

+ +

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a +room {{ bracketed_room_name_forhtml }} on Element.

+ +

+ Join the conversation. +

+ +

Element is an open source collaboration app built on the Matrix.org +open standard for interoperable communication: supporting group chat, file +transfer, voice and video calling, integrations to other apps, bridges +to other communication systems and much more.

+ +

Please note that Element requires +Chrome, +Firefox or +Safari on the web, +or iOS or Android on mobile.

+ +

Thanks,

+ +

Element

+ +
+

About Element:

+ +

Break through - Element allows teams to communicate across a wide range of collaboration +apps. If some team members use Element while others use IRC, Slack or Gitter, Element will +allow these team members to seamlessly work together. Element offers the richest +network of communication bridges.

+ +

Own Your Own Data - No one should control your communication and data but you. Element +lets you run your own server, and provides users and teams with the most advanced +crypto ratchet technology available today for a decentralized secure Internet.

+ +

Open Source - Element is entirely open source: all the code is published on GitHub +(Apache License) for anyone to see and extend. This means teams can customize or +contribute to the code and everyone can benefit from the speed of community innovation.

+ +

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, +decentralized communication delivering a community of users, bridged networks, +integrated bots and applications plus full end-to-end encryption. To learn more about +Matrix visit https://matrix.org.

+ +
+ + + +--{{ multipart_boundary }}-- diff --git a/res/vector-im/verification_template.eml.j2 b/res/vector-im/verification_template.eml.j2 new file mode 100644 index 00000000..01e592f7 --- /dev/null +++ b/res/vector-im/verification_template.eml.j2 @@ -0,0 +1,170 @@ +Date: {{ date }} +From: {{ from }} +To: {{ to }} +Message-ID: {{ messageid }} +Subject: Confirm your email address for Element +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="{{ multipart_boundary }}" + +--{{ multipart_boundary }} +Content-Type: text/plain; charset=UTF-8 +Content-Disposition: inline + +Hello there! + +You have asked us to register this email address with element.io - the open source, +distributed and secure shared workspace for the web that's built on Matrix. + +If it was really you who made this request, you can click on the following link to +complete the verification of your email address: + +{{ link }} + +Please note that you will need to use Chrome, Firefox or Safari on the web, or +iOS or Android on mobile. + +If you didn't make this request, you can safely disregard this email. + +Thanks! + +Element + + +About Element: + +Break through - Element allows teams to communicate across a wide range of collaboration +apps. If some team members use Element while others use IRC, Slack or Gitter, Element will +allow these team members to seamlessly work together. Element offers the richest +network of communication bridges. + +Own Your Own Data - No one should control your communication and data but you. Element +lets you run your own server, and provides users and teams with the most advanced +crypto ratchet technology available today for a decentralized secure Internet. + +Open Source - Element is entirely open source: all the code is published on GitHub +(Apache License) for anyone to see and extend. This means teams can customize or +contribute to the code and everyone can benefit from the speed of community innovation. + +Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, +decentralized communication delivering a community of users, bridged networks, +integrated bots and applications plus full end-to-end encryption. To learn more about +Matrix visit https://matrix.org. + +--{{ multipart_boundary }} +Content-Type: text/html; charset=UTF-8 +Content-Disposition: inline + + + + + + + + + + + + + +
+ + + + + +
+
+ +

Hello there!

+ +

You have asked us to register this email address with element.io - the open source, + distributed and secure shared workspace for the web that's built on Matrix.

+ +

If it was really you who made this request, you can click on the following link to + complete the verification of your email address:

+ +

Complete email verification

+ +

Please note that Element requires + Chrome, + Firefox or + Safari on the web, + or iOS or Android on mobile.

+ +

If you didn't make this request, you can safely disregard this email.

+ +

Thanks!

+ +

Element

+ +
+

About Element:

+ +

Break through - Element allows teams to communicate across a wide range of collaboration + apps. If some team members use Element while others use IRC, Slack or Gitter, Element will + allow these team members to seamlessly work together. Element offers the richest + network of communication bridges.

+ +

Own Your Own Data - No one should control your communication and data but you. Element + lets you run your own server, and provides users and teams with the most advanced + crypto ratchet technology available today for a decentralized secure Internet.

+ +

Open Source - Element is entirely open source: all the code is published on GitHub + (Apache License) for anyone to see and extend. This means teams can customize or + contribute to the code and everyone can benefit from the speed of community innovation.

+ +

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, + decentralized communication delivering a community of users, bridged networks, + integrated bots and applications plus full end-to-end encryption. To learn more about + Matrix visit https://matrix.org.

+ +
+ + + +--{{ multipart_boundary }}-- From 35eea479e548e0d09f50a51864874b0aa04749bf Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 09:21:03 -0700 Subject: [PATCH 07/38] check if jinja templates available and if so use them --- .../res/is-test/invite_template.eml | 12 ++++---- .../res/is-test/verification_template.eml | 2 +- res/matrix-org/invite_template.eml | 26 ++++++++--------- res/matrix-org/verification_template.eml | 26 ++++++++--------- res/vector-im/invite_template.eml | 28 +++++++++---------- res/vector-im/verification_template.eml | 20 ++++++------- sydent/sydent.py | 7 ++++- sydent/util/emailutils.py | 10 +++++-- 8 files changed, 70 insertions(+), 61 deletions(-) diff --git a/matrix_is_test/res/is-test/invite_template.eml b/matrix_is_test/res/is-test/invite_template.eml index 0b09cb12..557a88cb 100644 --- a/matrix_is_test/res/is-test/invite_template.eml +++ b/matrix_is_test/res/is-test/invite_template.eml @@ -1,8 +1,8 @@ { - "token": "{{ token }}", - "room_alias": "{{ room_alias }}", - "room_avatar_url": "{{ room_avatar_url }}", - "room_name": "{{ room_name }}", - "sender_display_name": "{{ sender_display_name }}", - "sender_avatar_url": "{{ sender_avatar_url }}" + "token": "%(token)s", + "room_alias": "%(room_alias)s", + "room_avatar_url": "%(room_avatar_url)s", + "room_name": "%(room_name)s", + "sender_display_name": "%(sender_display_name)s", + "sender_avatar_url": "%(sender_avatar_url)s" } diff --git a/matrix_is_test/res/is-test/verification_template.eml b/matrix_is_test/res/is-test/verification_template.eml index d74f9a83..22b0d2a1 100644 --- a/matrix_is_test/res/is-test/verification_template.eml +++ b/matrix_is_test/res/is-test/verification_template.eml @@ -1 +1 @@ -<<<{{ token }}>>> +<<<%(token)>>> diff --git a/res/matrix-org/invite_template.eml b/res/matrix-org/invite_template.eml index 579075d1..cfbbf667 100644 --- a/res/matrix-org/invite_template.eml +++ b/res/matrix-org/invite_template.eml @@ -1,20 +1,20 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} -Subject: {{ subject_header_value }} +Date: %(date)s +From: %(from)s +To: %(to)s +Message-ID: %(messageid)s +Subject: %(subject_header_value)s MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="%(multipart_boundary)s" ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room -{{ bracketed_room_name }}on Matrix. To join the conversation, either pick a +%(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room +%(bracketed_room_name)son Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element requires Chrome, Firefox, Safari, iOS or Android @@ -40,7 +40,7 @@ Thanks, Matrix ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -105,7 +105,7 @@ pre, code {

Hi,

-

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a room {{ bracketed_room_name_forhtml }} on +

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a room %(bracketed_room_name_forhtml)s on Matrix. To join the conversation, either pick a Matrix client or use the single-click link below to join via Element requires Chrome, @@ -116,7 +116,7 @@ or iOS or Android on mobile.

Join the conversation. + href="https://app.element.io/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s">Join the conversation.


@@ -143,4 +143,4 @@ create new communication solutions or extend the capabilities and reach of exist ---{{ multipart_boundary }}-- +--%(multipart_boundary)s-- diff --git a/res/matrix-org/verification_template.eml b/res/matrix-org/verification_template.eml index a9c7261e..d98e4d9e 100644 --- a/res/matrix-org/verification_template.eml +++ b/res/matrix-org/verification_template.eml @@ -1,13 +1,13 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} +Date: %(date)s +From: %(from)s +To: %(to)s +Message-ID: %(messageid)s Subject: Confirm your email address for Matrix MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="%(multipart_boundary)s" ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -17,9 +17,9 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -{{ link }} +%(link)s -If your client requires a code, the code is {{ token }} +If your client requires a code, the code is %(token)s If you aren't aware of making such a request, please disregard this email. @@ -35,7 +35,7 @@ Matrix defines the standard, and provides open source reference implementations Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -59,13 +59,13 @@ body { identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

...or copy this link into your web browser:

-

{{ link }}

+

%(link)s

-

If your client requires a code, the code is {{ token }}

+

If your client requires a code, the code is %(token)s

If you aren't aware of making such a request, please disregard this email.

@@ -85,4 +85,4 @@ create new communication solutions or extend the capabilities and reach of exist ---{{ multipart_boundary }}-- +--%(multipart_boundary)s-- diff --git a/res/vector-im/invite_template.eml b/res/vector-im/invite_template.eml index 23c9877a..1ca18025 100644 --- a/res/vector-im/invite_template.eml +++ b/res/vector-im/invite_template.eml @@ -1,20 +1,20 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} -Subject: {{ subject_header_value }} +Date: %(date)s +From: %(from)s +To: %(to)s +Message-ID: %(messageid)s +Subject: %(subject_header_value)s MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="%(multipart_boundary)s" ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room -{{ bracketed_room_name }}on Element. To join the conversation please follow the +%(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room +%(bracketed_room_name)son Element. To join the conversation please follow the link below. @@ -54,7 +54,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -125,12 +125,12 @@ pre, code {

Hi,

-

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a -room {{ bracketed_room_name_forhtml }} on Element.

+

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a +room %(bracketed_room_name_forhtml)s on Element.

Join the conversation. + href="https://app.element.io/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s">Join the conversation.

Element is an open source collaboration app built on the Matrix.org @@ -176,4 +176,4 @@ Matrix visit https://matrix.org.

---{{ multipart_boundary }}-- +--%(multipart_boundary)s-- diff --git a/res/vector-im/verification_template.eml b/res/vector-im/verification_template.eml index 01e592f7..e13b7e2c 100644 --- a/res/vector-im/verification_template.eml +++ b/res/vector-im/verification_template.eml @@ -1,13 +1,13 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} +Date: %(date)s +From: %(from)s +To: %(to)s +Message-ID: %(messageid)s Subject: Confirm your email address for Element MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="%(multipart_boundary)s" ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -19,7 +19,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -{{ link }} +%(link)s Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. @@ -51,7 +51,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---{{ multipart_boundary }} +--%(multipart_boundary)s Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -125,7 +125,7 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

Please note that Element requires Chrome, @@ -167,4 +167,4 @@ pre, code { ---{{ multipart_boundary }}-- +--%(multipart_boundary)s-- diff --git a/sydent/sydent.py b/sydent/sydent.py index e7579ca2..d7e24a00 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -472,7 +472,12 @@ def get_branded_template(self, brand, template_name, deprecated_template_name): brand = self.cfg.get("general", "brand.default") root_template_path = self.cfg.get("general", "templates.path") - return os.path.join(root_template_path, brand, template_name) + + # Grab jinja template if it exists + if os.path.exists(os.path.join(root_template_path, brand, template_name + ".j2")): + return os.path.join(root_template_path, brand, template_name + ".j2") + else: + return os.path.join(root_template_path, brand, template_name) class Validators: diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 9ac9a06e..6bf7add2 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -74,9 +74,13 @@ def sendEmail( # conflicting with it. allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) - with open(templateFile) as file_: - template = Template(file_.read()) - mailString = template.render(allSubstitutions) + # Render template with Jinja if using Jinja template + if templateFile.endswith('.j2'): + with open(templateFile) as file_: + template = Template(file_.read()) + mailString = template.render(allSubstitutions) + else: + mailString = open(templateFile).read() % allSubstitutions parsedFrom = email.utils.parseaddr(mailFrom)[1] parsedTo = email.utils.parseaddr(mailTo)[1] From 3f22e706d5a924b544449f5b14840ccf306f5b65 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 09:42:21 -0700 Subject: [PATCH 08/38] rename changelog file, version jinja in setup.py --- changelog.d/376.feature | 1 + changelog.d/376.misc | 1 - setup.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/376.feature delete mode 100644 changelog.d/376.misc diff --git a/changelog.d/376.feature b/changelog.d/376.feature new file mode 100644 index 00000000..5f76ac61 --- /dev/null +++ b/changelog.d/376.feature @@ -0,0 +1 @@ +Add support for using Jinja2 in e-mail templates. \ No newline at end of file diff --git a/changelog.d/376.misc b/changelog.d/376.misc deleted file mode 100644 index a7d5f34a..00000000 --- a/changelog.d/376.misc +++ /dev/null @@ -1 +0,0 @@ -Use jinja for templating emails. \ No newline at end of file diff --git a/setup.py b/setup.py index e1617d09..92a2c70f 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(fname): description="Reference Matrix Identity Verification and Lookup Server", python_requires=">=3.6", install_requires=[ - "jinja2", + "jinja2>=3.0.0", "signedjson==1.1.1", "unpaddedbase64==1.1.0", "Twisted>=18.4.0", From a97b0ae6bd5062d2ee98c3604852b2e402f05e34 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 09:43:30 -0700 Subject: [PATCH 09/38] lints --- sydent/sydent.py | 4 +++- sydent/util/emailutils.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sydent/sydent.py b/sydent/sydent.py index d7e24a00..6f370d3a 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -474,7 +474,9 @@ def get_branded_template(self, brand, template_name, deprecated_template_name): root_template_path = self.cfg.get("general", "templates.path") # Grab jinja template if it exists - if os.path.exists(os.path.join(root_template_path, brand, template_name + ".j2")): + if os.path.exists( + os.path.join(root_template_path, brand, template_name + ".j2") + ): return os.path.join(root_template_path, brand, template_name + ".j2") else: return os.path.join(root_template_path, brand, template_name) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 6bf7add2..4aef6200 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -75,7 +75,7 @@ def sendEmail( allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) # Render template with Jinja if using Jinja template - if templateFile.endswith('.j2'): + if templateFile.endswith(".j2"): with open(templateFile) as file_: template = Template(file_.read()) mailString = template.render(allSubstitutions) From 525be7245938f05f0dd10a707bb249165ffa7d8d Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 10:37:16 -0700 Subject: [PATCH 10/38] requested changes --- changelog.d/376.feature | 2 +- matrix_is_test/res/is-test/verification_template.eml | 2 +- res/matrix-org/invite_template.eml | 11 ++++------- res/matrix-org/invite_template.eml.j2 | 8 ++++---- sydent/util/emailutils.py | 10 +++++----- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/changelog.d/376.feature b/changelog.d/376.feature index 5f76ac61..d1ad46ee 100644 --- a/changelog.d/376.feature +++ b/changelog.d/376.feature @@ -1 +1 @@ -Add support for using Jinja2 in e-mail templates. \ No newline at end of file +Add support for using Jinja2 in e-mail templates. Contributed by H-Shay. \ No newline at end of file diff --git a/matrix_is_test/res/is-test/verification_template.eml b/matrix_is_test/res/is-test/verification_template.eml index 22b0d2a1..c75dd7dc 100644 --- a/matrix_is_test/res/is-test/verification_template.eml +++ b/matrix_is_test/res/is-test/verification_template.eml @@ -1 +1 @@ -<<<%(token)>>> +<<<%(token)s>>> diff --git a/res/matrix-org/invite_template.eml b/res/matrix-org/invite_template.eml index cfbbf667..9debb2f5 100644 --- a/res/matrix-org/invite_template.eml +++ b/res/matrix-org/invite_template.eml @@ -16,9 +16,8 @@ Hi, %(sender_display_name)s %(bracketed_verified_sender)shas invited you into a room %(bracketed_room_name)son Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use -the single-click link below to join via Element requires Chrome, Firefox, -Safari, iOS or Android - +the single-click link below to join via Element (requires Chrome, Firefox, +Safari, iOS or Android) %(web_client_location)s/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s @@ -84,8 +83,6 @@ pre, code { text-align: right; margin-left: 20px; } - - @@ -107,11 +104,11 @@ pre, code {

%(sender_display_name_forhtml)s %(bracketed_verified_sender_forhtml)s has invited you into a room %(bracketed_room_name_forhtml)s on Matrix. To join the conversation, either pick a Matrix client or use the single-click -link below to join via Element requires +link below to join via Element (requires Chrome, Firefox or Safari on the web, -or iOS or Android on mobile.

+or iOS or Android on mobile.)

diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index f1205651..aca3dfa8 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -16,8 +16,8 @@ Hi, {{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room {{ bracketed_room_name }}on Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use -the single-click link below to join via Element requires Chrome, Firefox, -Safari, iOS or Android +the single-click link below to join via Element (requires Chrome, Firefox, +Safari, iOS or Android) {{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} @@ -106,11 +106,11 @@ pre, code {

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a room {{ bracketed_room_name_forhtml }} on Matrix. To join the conversation, either pick a Matrix client or use the single-click -link below to join via Element requires +link below to join via Element (requires Chrome, Firefox or Safari on the web, -or iOS or Android on mobile.

+or iOS or Android on mobile.)

diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 4aef6200..8c545624 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -75,12 +75,12 @@ def sendEmail( allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) # Render template with Jinja if using Jinja template - if templateFile.endswith(".j2"): - with open(templateFile) as file_: - template = Template(file_.read()) + with open(templateFile) as template_file: + if templateFile.endswith(".j2"): + template = Template(template_file.read()) mailString = template.render(allSubstitutions) - else: - mailString = open(templateFile).read() % allSubstitutions + else: + mailString = template_file.read() % allSubstitutions parsedFrom = email.utils.parseaddr(mailFrom)[1] parsedTo = email.utils.parseaddr(mailTo)[1] From 308f9dcdd309a8ad55a75342f0c20ec6a9e67fc1 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 10:41:53 -0700 Subject: [PATCH 11/38] revoke change to terms.py --- sydent/terms/terms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index dc16108c..58bf83f1 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.safe_load(fp) + termsYaml = yaml.full_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: From a721485a4dd4709cbf05fcb7b79e31622ea19029 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 15:27:38 -0700 Subject: [PATCH 12/38] add some tests --- tests/test_templates.py | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/test_templates.py diff --git a/tests/test_templates.py b/tests/test_templates.py new file mode 100644 index 00000000..b94ef591 --- /dev/null +++ b/tests/test_templates.py @@ -0,0 +1,47 @@ +import os.path +from unittest.mock import patch + +from twisted.trial import unittest + +from tests.utils import make_sydent + +from sydent.util.emailutils import sendEmail + + +class TestTemplate(unittest.TestCase): + def setUp(self): + # Create a new sydent + config = { + "general": { + "templates.path": os.path.join( + os.path.dirname(os.path.dirname(__file__)), "res" + ), + }, + } + self.sydent = make_sydent(test_config=config) + + def test_jinja_template(self): + templateFile = self.sydent.get_branded_template("matrix", "invite_template.eml.j2", + ("email", "email.invite_template")) + substitutions = {"sender_display_name":"Betty Boop"} + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + self.assertIn("Betty Boop", email_contents) + + def test_jinja_escapes(self): + templateFile = self.sydent.get_branded_template("matrix", "invite_template.eml.j2", + ("email", "email.invite_template")) + substitutions = {"sender_display_name":""} + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + self.assertNotIn("", email_contents) From a54220100a02fc618bc80738530834d8fe4498d9 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 11 Aug 2021 15:28:38 -0700 Subject: [PATCH 13/38] use jinja environment and autoescape --- sydent/util/emailutils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 8c545624..e797a2ba 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -21,9 +21,10 @@ import urllib from html import escape from typing import TYPE_CHECKING, Dict +import os.path import twisted.python.log -from jinja2 import Template +from jinja2 import Template, Environment, select_autoescape, FileSystemLoader from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength @@ -75,9 +76,18 @@ def sendEmail( allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) # Render template with Jinja if using Jinja template + root_template_path = sydent.cfg.get("general", "templates.path") + env = Environment( + loader=FileSystemLoader(root_template_path), + autoescape=select_autoescape(['html', 'xml'])) + + # extract branded template name from templateFile + file_parts = templateFile.split('/') + template_name = file_parts[-2] + '/' + file_parts[-1] + with open(templateFile) as template_file: if templateFile.endswith(".j2"): - template = Template(template_file.read()) + template = env.get_template(template_name) mailString = template.render(allSubstitutions) else: mailString = template_file.read() % allSubstitutions From fb7db3c1e472046aa6cf51192eb879db3f4a5c12 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 12 Aug 2021 09:23:18 -0700 Subject: [PATCH 14/38] format jinja templates for autoescape and test --- res/matrix-org/invite_template.eml.j2 | 2 +- res/vector-im/invite_template.eml.j2 | 2 +- tests/test_templates.py | 41 +++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index aca3dfa8..d4d003e7 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -19,7 +19,7 @@ Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) -{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} +{{ web_client_location|e }}/#/room/{{ room_id_forurl|e }}?email={{ to_forurl|e }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token|e }}%%26private_key%%3D{{ ephemeral_private_key|e }}&room_name={{ room_name_forurl|e }}&room_avatar_url={{ room_avatar_url_forurl|e }}&inviter_name={{ sender_display_name_forurl|e }}&guest_access_token={{ guest_access_token_forurl|e }}&guest_user_id={{ guest_user_id_forurl|e }} About Matrix: diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 index 991b8f15..6a7a22e7 100644 --- a/res/vector-im/invite_template.eml.j2 +++ b/res/vector-im/invite_template.eml.j2 @@ -17,7 +17,7 @@ Hi, {{ bracketed_room_name }}on Element. To join the conversation please follow the link below. -{{ web_client_location)/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} +{{ web_client_location|e }}/#/room/{{ room_id_forurl|e }}?email={{ to_forurl|e }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token|e }}%%26private_key%%3D{{ ephemeral_private_key|e }}&room_name={{ room_name_forurl|e }}&room_avatar_url={{ room_avatar_url_forurl|e }}&inviter_name={{ sender_display_name_forurl|e }}&guest_access_token={{ guest_access_token_forurl|e }}&guest_user_id={{ guest_user_id_forurl|e }} Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, diff --git a/tests/test_templates.py b/tests/test_templates.py index b94ef591..b15e2dad 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -33,15 +33,52 @@ def test_jinja_template(self): self.assertIn("Betty Boop", email_contents) - def test_jinja_escapes(self): + def test_jinja_escapes_invite(self): + # test matrix invite template templateFile = self.sydent.get_branded_template("matrix", "invite_template.eml.j2", ("email", "email.invite_template")) - substitutions = {"sender_display_name":""} + substitutions = {"room_id_for_url":""} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + self.assertNotIn("", email_contents) + + # test vector-im invite template + templateFile = self.sydent.get_branded_template("vector-im", "invite_template.eml.j2", + ("email", "email.invite_template")) + substitutions = {"room_id_for_url":""} + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertNotIn("", email_contents) + + def test_jinja_escapes_verification(self): + # test matrix template + templateFile = self.sydent.get_branded_template("matrix", "verification_template.eml.j2", + ("email", "email.verification_template")) + substitutions = {"to":""} + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + self.assertNotIn("", email_contents) + + # test vector-im template + templateFile = self.sydent.get_branded_template("vector-im", "verification_template.eml.j2", + ("email", "email.verification_template")) + substitutions = {"to":""} + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + self.assertNotIn("", email_contents) \ No newline at end of file From 35c26405f8837e5dae778c56620db97915293da2 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 12 Aug 2021 09:38:21 -0700 Subject: [PATCH 15/38] fix tests, fix jinja templates --- res/matrix-org/invite_template.eml.j2 | 2 +- res/vector-im/invite_template.eml.j2 | 2 +- sydent/terms/terms.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index d4d003e7..aca3dfa8 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -19,7 +19,7 @@ Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) -{{ web_client_location|e }}/#/room/{{ room_id_forurl|e }}?email={{ to_forurl|e }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token|e }}%%26private_key%%3D{{ ephemeral_private_key|e }}&room_name={{ room_name_forurl|e }}&room_avatar_url={{ room_avatar_url_forurl|e }}&inviter_name={{ sender_display_name_forurl|e }}&guest_access_token={{ guest_access_token_forurl|e }}&guest_user_id={{ guest_user_id_forurl|e }} +{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} About Matrix: diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 index 6a7a22e7..50366e89 100644 --- a/res/vector-im/invite_template.eml.j2 +++ b/res/vector-im/invite_template.eml.j2 @@ -17,7 +17,7 @@ Hi, {{ bracketed_room_name }}on Element. To join the conversation please follow the link below. -{{ web_client_location|e }}/#/room/{{ room_id_forurl|e }}?email={{ to_forurl|e }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token|e }}%%26private_key%%3D{{ ephemeral_private_key|e }}&room_name={{ room_name_forurl|e }}&room_avatar_url={{ room_avatar_url_forurl|e }}&inviter_name={{ sender_display_name_forurl|e }}&guest_access_token={{ guest_access_token_forurl|e }}&guest_user_id={{ guest_user_id_forurl|e }} +{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index 58bf83f1..dc16108c 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.full_load(fp) + termsYaml = yaml.safe_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: From a3adf423198c3430d0afbc011ee2979ae92f64ec Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 12 Aug 2021 09:55:28 -0700 Subject: [PATCH 16/38] lints + update tests --- sydent/util/emailutils.py | 12 ++++---- tests/test_templates.py | 62 ++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index e797a2ba..385bc451 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -21,10 +21,9 @@ import urllib from html import escape from typing import TYPE_CHECKING, Dict -import os.path import twisted.python.log -from jinja2 import Template, Environment, select_autoescape, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, select_autoescape from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength @@ -78,12 +77,13 @@ def sendEmail( # Render template with Jinja if using Jinja template root_template_path = sydent.cfg.get("general", "templates.path") env = Environment( - loader=FileSystemLoader(root_template_path), - autoescape=select_autoescape(['html', 'xml'])) + loader=FileSystemLoader(root_template_path), + autoescape=select_autoescape(["html", "xml"]), + ) # extract branded template name from templateFile - file_parts = templateFile.split('/') - template_name = file_parts[-2] + '/' + file_parts[-1] + file_parts = templateFile.split("/") + template_name = file_parts[-2] + "/" + file_parts[-1] with open(templateFile) as template_file: if templateFile.endswith(".j2"): diff --git a/tests/test_templates.py b/tests/test_templates.py index b15e2dad..1470af3a 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -3,9 +3,8 @@ from twisted.trial import unittest -from tests.utils import make_sydent - from sydent.util.emailutils import sendEmail +from tests.utils import make_sydent class TestTemplate(unittest.TestCase): @@ -21,9 +20,11 @@ def setUp(self): self.sydent = make_sydent(test_config=config) def test_jinja_template(self): - templateFile = self.sydent.get_branded_template("matrix", "invite_template.eml.j2", - ("email", "email.invite_template")) - substitutions = {"sender_display_name":"Betty Boop"} + # test matrix invite template + templateFile = self.sydent.get_branded_template( + "matrix", "invite_template.eml.j2", ("email", "email.invite_template") + ) + substitutions = {"sender_display_name": "Betty Boop"} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) @@ -33,36 +34,28 @@ def test_jinja_template(self): self.assertIn("Betty Boop", email_contents) - def test_jinja_escapes_invite(self): - # test matrix invite template - templateFile = self.sydent.get_branded_template("matrix", "invite_template.eml.j2", - ("email", "email.invite_template")) - substitutions = {"room_id_for_url":""} + # test vector-im verification template + templateFile = self.sydent.get_branded_template( + "vector-im", + "verification_template.eml.j2", + ("email", "email.verification_template"), + ) + substitutions = {} with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + sendEmail(self.sydent, templateFile, "mickey@mouse.org", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertNotIn("", email_contents) - - # test vector-im invite template - templateFile = self.sydent.get_branded_template("vector-im", "invite_template.eml.j2", - ("email", "email.invite_template")) - substitutions = {"room_id_for_url":""} - with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + self.assertIn("mickey@mouse.org", email_contents) - smtp = smtplib.SMTP.return_value - email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertNotIn("", email_contents) - - def test_jinja_escapes_verification(self): - # test matrix template - templateFile = self.sydent.get_branded_template("matrix", "verification_template.eml.j2", - ("email", "email.verification_template")) - substitutions = {"to":""} + def test_jinja_escapes_invite(self): + # test matrix invite template + templateFile = self.sydent.get_branded_template( + "matrix", "invite_template.eml.j2", ("email", "email.invite_template") + ) + substitutions = {"to": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) @@ -71,14 +64,17 @@ def test_jinja_escapes_verification(self): email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertNotIn("", email_contents) - # test vector-im template - templateFile = self.sydent.get_branded_template("vector-im", "verification_template.eml.j2", - ("email", "email.verification_template")) - substitutions = {"to":""} + # test vector-im verification template + templateFile = self.sydent.get_branded_template( + "vector-im", + "verification_template.eml.j2", + ("email", "email.verification_template"), + ) + substitutions = {"to": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertNotIn("", email_contents) \ No newline at end of file + self.assertNotIn("", email_contents) From dd46fae728688a5ceb67eaf62e04c4b3a2b82432 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 12 Aug 2021 10:00:15 -0700 Subject: [PATCH 17/38] reset terms --- sydent/terms/terms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index dc16108c..58bf83f1 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.safe_load(fp) + termsYaml = yaml.full_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: From 96580b9133386957b195f8303c2822190b0ba1ef Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 13 Aug 2021 10:03:57 -0700 Subject: [PATCH 18/38] update templates with safe filter --- res/matrix-org/invite_template.eml.j2 | 18 +++++++++--------- res/matrix-org/verification_template.eml.j2 | 16 ++++++++-------- res/vector-im/invite_template.eml.j2 | 20 ++++++++++---------- res/vector-im/verification_template.eml.j2 | 12 ++++++------ 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index aca3dfa8..f08d850e 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -1,20 +1,20 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} -Subject: {{ subject_header_value }} +Date: {{ date|safe }} +From: {{ from|safe }} +To: {{ to|safe }} +Message-ID: {{ messageid|safe }} +Subject: {{ subject_header_value|safe }} MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="{{ multipart_boundary|safe }}" ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room -{{ bracketed_room_name }}on Matrix. To join the conversation, either pick a +{{ sender_display_name|safe }} {{ bracketed_verified_sender|safe }}has invited you into a room +{{ bracketed_room_name|safe }}on Matrix. To join the conversation, either pick a Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) diff --git a/res/matrix-org/verification_template.eml.j2 b/res/matrix-org/verification_template.eml.j2 index a9c7261e..84d1d6a1 100644 --- a/res/matrix-org/verification_template.eml.j2 +++ b/res/matrix-org/verification_template.eml.j2 @@ -1,13 +1,13 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} +Date: {{ date|safe }} +From: {{ from|safe }} +To: {{ to|safe }} +Message-ID: {{ messageid|safe }} Subject: Confirm your email address for Matrix MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="{{ multipart_boundary|safe }}" ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -17,9 +17,9 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -{{ link }} +{{ link|safe }} -If your client requires a code, the code is {{ token }} +If your client requires a code, the code is {{ token|safe }} If you aren't aware of making such a request, please disregard this email. diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 index 50366e89..427b72e7 100644 --- a/res/vector-im/invite_template.eml.j2 +++ b/res/vector-im/invite_template.eml.j2 @@ -1,20 +1,20 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} -Subject: {{ subject_header_value }} +Date: {{ date|safe }} +From: {{ from|safe }} +To: {{ to|safe }} +Message-ID: {{ messageid|safe }} +Subject: {{ subject_header_value|safe }} MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="{{ multipart_boundary|safe }}" ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Hi, -{{ sender_display_name }} {{ bracketed_verified_sender }}has invited you into a room -{{ bracketed_room_name }}on Element. To join the conversation please follow the +{{ sender_display_name|safe }} {{ bracketed_verified_sender|safe }}has invited you into a room +{{ bracketed_room_name|safe }}on Element. To join the conversation please follow the link below. {{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} @@ -52,7 +52,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline diff --git a/res/vector-im/verification_template.eml.j2 b/res/vector-im/verification_template.eml.j2 index 01e592f7..57eab465 100644 --- a/res/vector-im/verification_template.eml.j2 +++ b/res/vector-im/verification_template.eml.j2 @@ -1,11 +1,11 @@ -Date: {{ date }} -From: {{ from }} -To: {{ to }} -Message-ID: {{ messageid }} +Date: {{ date|safe }} +From: {{ from|safe }} +To: {{ to|safe }} +Message-ID: {{ messageid|safe }} Subject: Confirm your email address for Element MIME-Version: 1.0 Content-Type: multipart/alternative; - boundary="{{ multipart_boundary }}" + boundary="{{ multipart_boundary|safe }}" --{{ multipart_boundary }} Content-Type: text/plain; charset=UTF-8 @@ -19,7 +19,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -{{ link }} +{{ link|safe }} Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. From c35fd944e4145c3d95e928167fc1227bc69d1c79 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 13 Aug 2021 10:05:25 -0700 Subject: [PATCH 19/38] move jinja environment creation to startup, add env to sydent --- sydent/sydent.py | 5 +++++ sydent/util/emailutils.py | 10 ++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sydent/sydent.py b/sydent/sydent.py index 6f370d3a..cc8db3bc 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -21,6 +21,7 @@ import logging.handlers import os from typing import Set +from jinja2 import Environment, FileSystemLoader, select_autoescape import twisted.internet.reactor from twisted.internet import address, task @@ -294,6 +295,10 @@ def __init__( "email", "email.third_party_invite_domain_obfuscate_characters" ) ) + self.template_environment = Environment( + loader=FileSystemLoader(self.cfg.get("general", "templates.path")), + autoescape=select_autoescape(default = True), + ) # See if a pepper already exists in the database # Note: This MUST be run before we start serving requests, otherwise lookups for diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 385bc451..d7fe8c82 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -74,20 +74,14 @@ def sendEmail( # conflicting with it. allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) - # Render template with Jinja if using Jinja template - root_template_path = sydent.cfg.get("general", "templates.path") - env = Environment( - loader=FileSystemLoader(root_template_path), - autoescape=select_autoescape(["html", "xml"]), - ) - # extract branded template name from templateFile file_parts = templateFile.split("/") template_name = file_parts[-2] + "/" + file_parts[-1] + # use jinja for rendering if jinja templates are present with open(templateFile) as template_file: if templateFile.endswith(".j2"): - template = env.get_template(template_name) + template = sydent.template_environment.get_template(template_name) mailString = template.render(allSubstitutions) else: mailString = template_file.read() % allSubstitutions From 56e0750dac19bb43cd2a6d91a185f301db8f39db Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 13 Aug 2021 10:05:57 -0700 Subject: [PATCH 20/38] update tests --- tests/test_templates.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index 1470af3a..48252f5a 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -4,7 +4,7 @@ from twisted.trial import unittest from sydent.util.emailutils import sendEmail -from tests.utils import make_sydent +from tests.utils import make_sydent, make_request class TestTemplate(unittest.TestCase): @@ -51,11 +51,10 @@ def test_jinja_template(self): self.assertIn("mickey@mouse.org", email_contents) def test_jinja_escapes_invite(self): - # test matrix invite template templateFile = self.sydent.get_branded_template( "matrix", "invite_template.eml.j2", ("email", "email.invite_template") ) - substitutions = {"to": ""} + substitutions = {"sender_display_name_forhtml": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) @@ -64,17 +63,17 @@ def test_jinja_escapes_invite(self): email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") self.assertNotIn("", email_contents) - # test vector-im verification template + def test_jinja_does_not_escape_safe_values(self): templateFile = self.sydent.get_branded_template( "vector-im", "verification_template.eml.j2", ("email", "email.verification_template"), ) - substitutions = {"to": ""} + substitutions = {"link": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertNotIn("", email_contents) + self.assertIn("", email_contents) From 4606fdebb6dc9712b875af1e95e8e6aeaa5ca0cd Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 13 Aug 2021 10:11:22 -0700 Subject: [PATCH 21/38] lints --- sydent/sydent.py | 4 ++-- sydent/terms/terms.py | 2 +- sydent/util/emailutils.py | 1 - tests/test_templates.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sydent/sydent.py b/sydent/sydent.py index cc8db3bc..ee629b76 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -21,9 +21,9 @@ import logging.handlers import os from typing import Set -from jinja2 import Environment, FileSystemLoader, select_autoescape import twisted.internet.reactor +from jinja2 import Environment, FileSystemLoader, select_autoescape from twisted.internet import address, task from twisted.python import log @@ -297,7 +297,7 @@ def __init__( ) self.template_environment = Environment( loader=FileSystemLoader(self.cfg.get("general", "templates.path")), - autoescape=select_autoescape(default = True), + autoescape=select_autoescape(default=True), ) # See if a pepper already exists in the database diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index 58bf83f1..dc16108c 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.full_load(fp) + termsYaml = yaml.safe_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index d7fe8c82..a50ccf32 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -23,7 +23,6 @@ from typing import TYPE_CHECKING, Dict import twisted.python.log -from jinja2 import Environment, FileSystemLoader, select_autoescape from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength diff --git a/tests/test_templates.py b/tests/test_templates.py index 48252f5a..4c9867ca 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -4,7 +4,7 @@ from twisted.trial import unittest from sydent.util.emailutils import sendEmail -from tests.utils import make_sydent, make_request +from tests.utils import make_sydent class TestTemplate(unittest.TestCase): From a5f14165cac751f61ca192ca850985c953f23b67 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 16 Aug 2021 11:14:56 -0700 Subject: [PATCH 22/38] update template files --- res/matrix-org/invite_template.eml.j2 | 4 ++-- res/matrix-org/verification_template.eml.j2 | 8 ++++---- res/vector-im/invite_template.eml.j2 | 2 +- res/vector-im/verification_template.eml.j2 | 9 +++++---- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index f08d850e..127b9134 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -39,7 +39,7 @@ Thanks, Matrix ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -142,4 +142,4 @@ create new communication solutions or extend the capabilities and reach of exist ---{{ multipart_boundary }}-- +--{{ multipart_boundary|safe }}-- diff --git a/res/matrix-org/verification_template.eml.j2 b/res/matrix-org/verification_template.eml.j2 index 84d1d6a1..9df9b06b 100644 --- a/res/matrix-org/verification_template.eml.j2 +++ b/res/matrix-org/verification_template.eml.j2 @@ -35,7 +35,7 @@ Matrix defines the standard, and provides open source reference implementations Matrix-compatible Servers, Clients, Client SDKs and Application Services to help you create new communication solutions or extend the capabilities and reach of existing ones. ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -59,11 +59,11 @@ body { identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

...or copy this link into your web browser:

-

{{ link }}

+

{{ link|safe }}

If your client requires a code, the code is {{ token }}

@@ -85,4 +85,4 @@ create new communication solutions or extend the capabilities and reach of exist ---{{ multipart_boundary }}-- +--{{ multipart_boundary|safe }}-- diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 index 427b72e7..2b169503 100644 --- a/res/vector-im/invite_template.eml.j2 +++ b/res/vector-im/invite_template.eml.j2 @@ -174,4 +174,4 @@ Matrix visit https://matrix.org.

---{{ multipart_boundary }}-- +--{{ multipart_boundary|safe }}-- diff --git a/res/vector-im/verification_template.eml.j2 b/res/vector-im/verification_template.eml.j2 index 57eab465..20c4eda8 100644 --- a/res/vector-im/verification_template.eml.j2 +++ b/res/vector-im/verification_template.eml.j2 @@ -7,7 +7,7 @@ MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="{{ multipart_boundary|safe }}" ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline @@ -51,7 +51,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. ---{{ multipart_boundary }} +--{{ multipart_boundary|safe }} Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -125,7 +125,8 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

+

Please note that Element requires Chrome, @@ -167,4 +168,4 @@ pre, code { ---{{ multipart_boundary }}-- +--{{ multipart_boundary|safe }}-- From 808b539944a6d6aaaadcd044a38a8df3b9eee38a Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 16 Aug 2021 11:15:49 -0700 Subject: [PATCH 23/38] update tests and send_email function --- sydent/util/emailutils.py | 6 +++++- tests/test_templates.py | 9 ++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index a50ccf32..fa5a8293 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -64,9 +64,13 @@ def sendEmail( ) allSubstitutions = {} + for k, v in substitutions.items(): allSubstitutions[k] = v - allSubstitutions[k + "_forhtml"] = escape(v) + if templateFile.endswith(".j2"): + allSubstitutions[k + "_forhtml"] = v + else: + allSubstitutions[k + "_forhtml"] = escape(v) allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) # We add randomize the multipart boundary to stop user input from diff --git a/tests/test_templates.py b/tests/test_templates.py index 4c9867ca..546de30e 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -22,7 +22,7 @@ def setUp(self): def test_jinja_template(self): # test matrix invite template templateFile = self.sydent.get_branded_template( - "matrix", "invite_template.eml.j2", ("email", "email.invite_template") + "matrix", "invite_template.eml.j2", ("email", "email.invite_template.j2") ) substitutions = {"sender_display_name": "Betty Boop"} @@ -31,14 +31,13 @@ def test_jinja_template(self): smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertIn("Betty Boop", email_contents) # test vector-im verification template templateFile = self.sydent.get_branded_template( "vector-im", - "verification_template.eml.j2", - ("email", "email.verification_template"), + "verification_template.eml", + ("email", "email.verification_template.j2"), ) substitutions = {} @@ -50,7 +49,7 @@ def test_jinja_template(self): self.assertIn("mickey@mouse.org", email_contents) - def test_jinja_escapes_invite(self): + def test_jinja_escapes(self): templateFile = self.sydent.get_branded_template( "matrix", "invite_template.eml.j2", ("email", "email.invite_template") ) From f0f25cd0f9721a1ff3d2bbc5f6f2a60227c40a8a Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 16 Aug 2021 11:18:23 -0700 Subject: [PATCH 24/38] reset terms --- sydent/terms/terms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index dc16108c..58bf83f1 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -107,7 +107,7 @@ def get_terms(sydent) -> Optional[Terms]: return Terms(None) with open(termsPath) as fp: - termsYaml = yaml.safe_load(fp) + termsYaml = yaml.full_load(fp) if "master_version" not in termsYaml: raise Exception("No master version") if "docs" not in termsYaml: From e75c4d77a55fe1cb676e60ea69b2519811db14a4 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:04:37 -0700 Subject: [PATCH 25/38] update templates --- res/matrix-org/invite_template.eml.j2 | 7 ++++--- res/matrix-org/verification_template.eml.j2 | 6 +++--- res/vector-im/invite_template.eml.j2 | 8 ++++---- res/vector-im/verification_template.eml.j2 | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/res/matrix-org/invite_template.eml.j2 b/res/matrix-org/invite_template.eml.j2 index 127b9134..9fd23212 100644 --- a/res/matrix-org/invite_template.eml.j2 +++ b/res/matrix-org/invite_template.eml.j2 @@ -19,7 +19,8 @@ Matrix client from https://matrix.org/docs/projects/try-matrix-now.html or use the single-click link below to join via Element (requires Chrome, Firefox, Safari, iOS or Android) -{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fmatrix.org%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} + +{{ web_client_location }}/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }} About Matrix: @@ -104,7 +105,7 @@ pre, code {

Hi,

-

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a room {{ bracketed_room_name_forhtml }} on +

{{ sender_display_name }} {{ bracketed_verified_sender }} has invited you into a room {{ bracketed_room_name }} on Matrix. To join the conversation, either pick a Matrix client or use the single-click link below to join via Element (requires Chrome, @@ -115,7 +116,7 @@ or iOS or Android on mobile.)

Join the conversation. + href="https://app.element.io/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }}">Join the conversation.


diff --git a/res/matrix-org/verification_template.eml.j2 b/res/matrix-org/verification_template.eml.j2 index 9df9b06b..fc3e5240 100644 --- a/res/matrix-org/verification_template.eml.j2 +++ b/res/matrix-org/verification_template.eml.j2 @@ -17,7 +17,7 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -{{ link|safe }} +{{ link|urlencode }} If your client requires a code, the code is {{ token|safe }} @@ -59,11 +59,11 @@ body { identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

...or copy this link into your web browser:

-

{{ link|safe }}

+

{{ link|urlencode }}

If your client requires a code, the code is {{ token }}

diff --git a/res/vector-im/invite_template.eml.j2 b/res/vector-im/invite_template.eml.j2 index 2b169503..d35a4342 100644 --- a/res/vector-im/invite_template.eml.j2 +++ b/res/vector-im/invite_template.eml.j2 @@ -17,7 +17,7 @@ Hi, {{ bracketed_room_name|safe }}on Element. To join the conversation please follow the link below. -{{ web_client_location }}/#/room/{{ room_id_forurl }}?email={{ to_forurl }}&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D{{ token }}%%26private_key%%3D{{ ephemeral_private_key }}&room_name={{ room_name_forurl }}&room_avatar_url={{ room_avatar_url_forurl }}&inviter_name={{ sender_display_name_forurl }}&guest_access_token={{ guest_access_token_forurl }}&guest_user_id={{ guest_user_id_forurl }} +{{ web_client_location }}/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }} Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, @@ -123,12 +123,12 @@ pre, code {

Hi,

-

{{ sender_display_name_forhtml }} {{ bracketed_verified_sender_forhtml }} has invited you into a -room {{ bracketed_room_name_forhtml }} on Element.

+

{{ sender_display_name }} {{ bracketed_verified_sender }} has invited you into a +room {{ bracketed_room_name }} on Element.

Join the conversation. + href="https://app.element.io/#/room/{{ room_id|urlencode }}?email={{ to|urlencode }}&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D{{ token|urlencode }}%26private_key%3D{{ ephemeral_private_key|urlencode }}&room_name={{ room_name|urlencode }}&room_avatar_url={{ room_avatar_url|urlencode }}&inviter_name={{ sender_display_name|urlencode }}&guest_access_token={{ guest_access_token|urlencode }}&guest_user_id={{ guest_user_id|urlencode }}">Join the conversation.

Element is an open source collaboration app built on the Matrix.org diff --git a/res/vector-im/verification_template.eml.j2 b/res/vector-im/verification_template.eml.j2 index 20c4eda8..bb20ff2e 100644 --- a/res/vector-im/verification_template.eml.j2 +++ b/res/vector-im/verification_template.eml.j2 @@ -19,7 +19,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -{{ link|safe }} +{{ link|urlencode }} Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. @@ -125,7 +125,7 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

Please note that Element requires From ab9ccecdbc27cc97ffaa9f0214e8f6acfbd057c2 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:05:52 -0700 Subject: [PATCH 26/38] only manually escape old templates --- sydent/util/emailutils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index fa5a8293..1802697d 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -67,11 +67,9 @@ def sendEmail( for k, v in substitutions.items(): allSubstitutions[k] = v - if templateFile.endswith(".j2"): - allSubstitutions[k + "_forhtml"] = v - else: + if not templateFile.endswith(".j2"): allSubstitutions[k + "_forhtml"] = escape(v) - allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) + allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) # We add randomize the multipart boundary to stop user input from # conflicting with it. From 5c8dc2e6569c5a7d1cc75673eca19115b4ed9dc9 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:08:17 -0700 Subject: [PATCH 27/38] update tests --- tests/test_templates.py | 155 ++++++++++++++++++++++++++++++++++------ 1 file changed, 132 insertions(+), 23 deletions(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index 546de30e..d37afc83 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -4,7 +4,9 @@ from twisted.trial import unittest from sydent.util.emailutils import sendEmail -from tests.utils import make_sydent +from tests.utils import make_sydent, make_request + +import urllib class TestTemplate(unittest.TestCase): @@ -19,60 +21,167 @@ def setUp(self): } self.sydent = make_sydent(test_config=config) - def test_jinja_template(self): - # test matrix invite template + + def test_jinja_vector_invite(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender" : "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "web_client_location" : "element", + "to" : "person@test.test", + "token": "a_token", + "ephemeral_private_key" : "mystery_key", + "web_client_location" : "https://app.element.io" + } + templateFile = self.sydent.get_branded_template( - "matrix", "invite_template.eml.j2", ("email", "email.invite_template.j2") + "vector-im", + "invite_template.eml.j2", + ("email", "email.invite_template"), ) - substitutions = {"sender_display_name": "Betty Boop"} with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertIn("Betty Boop", email_contents) - # test vector-im verification template + # test url input is encoded + self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) + + # test html input is escaped + self.assertIn("Bob's Emporium of Messages", email_contents) + + # test safe values are not escaped + self.assertIn("", email_contents) + + #test our link is as expected + expected_url = 'https://app.element.io/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + text = email_contents.splitlines() + link = text[19] + self.assertEqual(link, expected_url) + + def test_jinja_matrix_invite(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender" : "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "web_client_location" : "element", + "to" : "person@test.test", + "token": "a_token", + "ephemeral_private_key" : "mystery_key", + "web_client_location" : "https://matrix.org" + } + templateFile = self.sydent.get_branded_template( - "vector-im", - "verification_template.eml", - ("email", "email.verification_template.j2"), + "matrix-org", + "invite_template.eml.j2", + ("email", "email.invite_template"), ) - substitutions = {} with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "mickey@mouse.org", substitutions) + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertIn("mickey@mouse.org", email_contents) + # test url input is encoded + self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) + + # test html input is escaped + self.assertIn("Bob's Emporium of Messages", email_contents) + + # test safe values are not escaped + self.assertIn("", email_contents) + + #test our link is as expected + expected_url = 'https://matrix.org/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + text = email_contents.splitlines() + link = text[22] + self.assertEqual(link, expected_url) + + + def test_jinja_matrix_verification(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "to" : "person@test.test", + "token": "<>", + "link" : "https://link_test.com" + } - def test_jinja_escapes(self): templateFile = self.sydent.get_branded_template( - "matrix", "invite_template.eml.j2", ("email", "email.invite_template") + "matrix-org", + "verification_template.eml.j2", + ("email", "email.verification_template"), ) - substitutions = {"sender_display_name_forhtml": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertNotIn("", email_contents) - def test_jinja_does_not_escape_safe_values(self): + # test url input is encoded + self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) + + # test html input is escaped + self.assertIn("<<token>>", email_contents) + + # test safe values are not escaped + self.assertIn("<>", email_contents) + + + def test_jinja_vector_verification(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "to" : "person@test.test", + "token": "<>", + "link" : "https://link_test.com" + } + templateFile = self.sydent.get_branded_template( "vector-im", "verification_template.eml.j2", ("email", "email.verification_template"), ) - substitutions = {"link": ""} with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, "blah@nowhere.com", substitutions) + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - self.assertIn("", email_contents) + + # test url input is encoded + self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) + + # test all output is as expected + path = os.path.join(self.sydent.cfg.get("general", "templates.path"), 'vector_verification_sample.txt') + + with open(path, 'r') as file: + expected_text = file.read() + + # remove headers as they are variable + email_contents = email_contents.splitlines()[13:] + + # remove multipart headers as they are variable + del email_contents[40] + del email_contents[156] + self.assertEqual("\n".join(email_contents), expected_text) From e43f0f93e424e6763dda05c77759718f5243fcd1 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:09:24 -0700 Subject: [PATCH 28/38] add smaple text for testing --- res/vector_verification_sample.txt | 155 +++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 res/vector_verification_sample.txt diff --git a/res/vector_verification_sample.txt b/res/vector_verification_sample.txt new file mode 100644 index 00000000..34fdbc94 --- /dev/null +++ b/res/vector_verification_sample.txt @@ -0,0 +1,155 @@ +Hello there! + +You have asked us to register this email address with element.io - the open source, +distributed and secure shared workspace for the web that's built on Matrix. + +If it was really you who made this request, you can click on the following link to +complete the verification of your email address: + +https%3A//link_test.com + +Please note that you will need to use Chrome, Firefox or Safari on the web, or +iOS or Android on mobile. + +If you didn't make this request, you can safely disregard this email. + +Thanks! + +Element + + +About Element: + +Break through - Element allows teams to communicate across a wide range of collaboration +apps. If some team members use Element while others use IRC, Slack or Gitter, Element will +allow these team members to seamlessly work together. Element offers the richest +network of communication bridges. + +Own Your Own Data - No one should control your communication and data but you. Element +lets you run your own server, and provides users and teams with the most advanced +crypto ratchet technology available today for a decentralized secure Internet. + +Open Source - Element is entirely open source: all the code is published on GitHub +(Apache License) for anyone to see and extend. This means teams can customize or +contribute to the code and everyone can benefit from the speed of community innovation. + +Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, +decentralized communication delivering a community of users, bridged networks, +integrated bots and applications plus full end-to-end encryption. To learn more about +Matrix visit https://matrix.org. + +Content-Type: text/html; charset=UTF-8 +Content-Disposition: inline + + + + + + + + + + + + + +
+ + + + + +
+
+ +

Hello there!

+ +

You have asked us to register this email address with element.io - the open source, + distributed and secure shared workspace for the web that's built on Matrix.

+ +

If it was really you who made this request, you can click on the following link to + complete the verification of your email address:

+ +

Complete email verification

+ + +

Please note that Element requires + Chrome, + Firefox or + Safari on the web, + or iOS or Android on mobile.

+ +

If you didn't make this request, you can safely disregard this email.

+ +

Thanks!

+ +

Element

+ +
+

About Element:

+ +

Break through - Element allows teams to communicate across a wide range of collaboration + apps. If some team members use Element while others use IRC, Slack or Gitter, Element will + allow these team members to seamlessly work together. Element offers the richest + network of communication bridges.

+ +

Own Your Own Data - No one should control your communication and data but you. Element + lets you run your own server, and provides users and teams with the most advanced + crypto ratchet technology available today for a decentralized secure Internet.

+ +

Open Source - Element is entirely open source: all the code is published on GitHub + (Apache License) for anyone to see and extend. This means teams can customize or + contribute to the code and everyone can benefit from the speed of community innovation.

+ +

Made on Matrix - Element is built on top of Matrix. Matrix is an open network for secure, + decentralized communication delivering a community of users, bridged networks, + integrated bots and applications plus full end-to-end encryption. To learn more about + Matrix visit https://matrix.org.

+ +
+ + From 3e3b11624c4f6bf4e1b40cda78b5fcb69d376826 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:19:12 -0700 Subject: [PATCH 29/38] update tests name --- tests/test_jinja_templates.py | 187 ++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/test_jinja_templates.py diff --git a/tests/test_jinja_templates.py b/tests/test_jinja_templates.py new file mode 100644 index 00000000..e86e5fcb --- /dev/null +++ b/tests/test_jinja_templates.py @@ -0,0 +1,187 @@ +import os.path +from unittest.mock import patch + +from twisted.trial import unittest + +from sydent.util.emailutils import sendEmail +from tests.utils import make_sydent, make_request + +import urllib + + +class TestTemplate(unittest.TestCase): + def setUp(self): + # Create a new sydent + config = { + "general": { + "templates.path": os.path.join( + os.path.dirname(os.path.dirname(__file__)), "res" + ), + }, + } + self.sydent = make_sydent(test_config=config) + + + def test_jinja_vector_invite(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender" : "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "web_client_location" : "element", + "to" : "person@test.test", + "token": "a_token", + "ephemeral_private_key" : "mystery_key", + "web_client_location" : "https://app.element.io" + } + + templateFile = self.sydent.get_branded_template( + "vector-im", + "invite_template.eml", + ("email", "email.invite_template"), + ) + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + # test url input is encoded + self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) + + # test html input is escaped + self.assertIn("Bob's Emporium of Messages", email_contents) + + # test safe values are not escaped + self.assertIn("", email_contents) + + #test our link is as expected + expected_url = 'https://app.element.io/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + text = email_contents.splitlines() + link = text[19] + self.assertEqual(link, expected_url) + + def test_jinja_matrix_invite(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender" : "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "web_client_location" : "element", + "to" : "person@test.test", + "token": "a_token", + "ephemeral_private_key" : "mystery_key", + "web_client_location" : "https://matrix.org" + } + + templateFile = self.sydent.get_branded_template( + "matrix-org", + "invite_template.eml", + ("email", "email.invite_template"), + ) + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + # test url input is encoded + self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) + + # test html input is escaped + self.assertIn("Bob's Emporium of Messages", email_contents) + + # test safe values are not escaped + self.assertIn("", email_contents) + + #test our link is as expected + expected_url = 'https://matrix.org/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + text = email_contents.splitlines() + link = text[22] + self.assertEqual(link, expected_url) + + + def test_jinja_matrix_verification(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "to" : "person@test.test", + "token": "<>", + "link" : "https://link_test.com" + } + + templateFile = self.sydent.get_branded_template( + "matrix-org", + "verification_template.eml", + ("email", "email.verification_template"), + ) + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + # test url input is encoded + self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) + + # test html input is escaped + self.assertIn("<<token>>", email_contents) + + # test safe values are not escaped + self.assertIn("<>", email_contents) + + + def test_jinja_vector_verification(self): + substitutions = { + "address": "foo@example.com", + "medium": "email", + "to" : "person@test.test", + "token": "<>", + "link" : "https://link_test.com" + } + + templateFile = self.sydent.get_branded_template( + "vector-im", + "verification_template.eml", + ("email", "email.verification_template"), + ) + + with patch("sydent.util.emailutils.smtplib") as smtplib: + sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + + smtp = smtplib.SMTP.return_value + email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") + + # test url input is encoded + self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) + + # test all output is as expected + path = os.path.join(self.sydent.cfg.get("general", "templates.path"), 'vector_verification_sample.txt') + + with open(path, 'r') as file: + expected_text = file.read() + + # remove headers as they are variable + email_contents = email_contents.splitlines()[13:] + + # remove multipart headers as they are variable + del email_contents[40] + del email_contents[156] + self.assertEqual("\n".join(email_contents), expected_text) From 6c60693106fd9c6b269a0e9c987cd99760b48728 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:39:23 -0700 Subject: [PATCH 30/38] lints + rename jinja tests --- tests/test_jinja_templates.py | 155 ++++++++++++++++------------ tests/test_templates.py | 187 ---------------------------------- 2 files changed, 92 insertions(+), 250 deletions(-) delete mode 100644 tests/test_templates.py diff --git a/tests/test_jinja_templates.py b/tests/test_jinja_templates.py index e86e5fcb..c2bcc590 100644 --- a/tests/test_jinja_templates.py +++ b/tests/test_jinja_templates.py @@ -1,12 +1,11 @@ import os.path +import urllib from unittest.mock import patch from twisted.trial import unittest from sydent.util.emailutils import sendEmail -from tests.utils import make_sydent, make_request - -import urllib +from tests.utils import make_sydent class TestTemplate(unittest.TestCase): @@ -21,26 +20,24 @@ def setUp(self): } self.sydent = make_sydent(test_config=config) - def test_jinja_vector_invite(self): substitutions = { - "address": "foo@example.com", - "medium": "email", - "room_alias": "#somewhere:exmaple.org", - "room_avatar_url": "mxc://example.org/s0meM3dia", - "room_id": "!something:example.org", - "room_name": "Bob's Emporium of Messages", - "sender": "@bob:example.com", - "sender_avatar_url": "mxc://example.org/an0th3rM3dia", - "sender_display_name": "", - "bracketed_verified_sender" : "Bob Smith", - "bracketed_room_name": "Bob's Emporium of Messages", - "web_client_location" : "element", - "to" : "person@test.test", - "token": "a_token", - "ephemeral_private_key" : "mystery_key", - "web_client_location" : "https://app.element.io" - } + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender": "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "to": "person@test.test", + "token": "a_token", + "ephemeral_private_key": "mystery_key", + "web_client_location": "https://app.element.io", + } templateFile = self.sydent.get_branded_template( "vector-im", @@ -49,7 +46,7 @@ def test_jinja_vector_invite(self): ) with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") @@ -63,31 +60,46 @@ def test_jinja_vector_invite(self): # test safe values are not escaped self.assertIn("", email_contents) - #test our link is as expected - expected_url = 'https://app.element.io/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + # test our link is as expected + expected_url = ( + "https://app.element.io/#/room/" + + urllib.parse.quote("!something:example.org") + + "?email=" + + urllib.parse.quote("test@test.com") + + "&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D" + + urllib.parse.quote("a_token") + + "%26private_key%3D" + + urllib.parse.quote("mystery_key") + + "&room_name=" + + urllib.parse.quote("Bob's Emporium of Messages") + + "&room_avatar_url=" + + urllib.parse.quote("mxc://example.org/s0meM3dia") + + "&inviter_name=" + + urllib.parse.quote("") + + "&guest_access_token=&guest_user_id=" + ) text = email_contents.splitlines() link = text[19] self.assertEqual(link, expected_url) def test_jinja_matrix_invite(self): substitutions = { - "address": "foo@example.com", - "medium": "email", - "room_alias": "#somewhere:exmaple.org", - "room_avatar_url": "mxc://example.org/s0meM3dia", - "room_id": "!something:example.org", - "room_name": "Bob's Emporium of Messages", - "sender": "@bob:example.com", - "sender_avatar_url": "mxc://example.org/an0th3rM3dia", - "sender_display_name": "", - "bracketed_verified_sender" : "Bob Smith", - "bracketed_room_name": "Bob's Emporium of Messages", - "web_client_location" : "element", - "to" : "person@test.test", - "token": "a_token", - "ephemeral_private_key" : "mystery_key", - "web_client_location" : "https://matrix.org" - } + "address": "foo@example.com", + "medium": "email", + "room_alias": "#somewhere:exmaple.org", + "room_avatar_url": "mxc://example.org/s0meM3dia", + "room_id": "!something:example.org", + "room_name": "Bob's Emporium of Messages", + "sender": "@bob:example.com", + "sender_avatar_url": "mxc://example.org/an0th3rM3dia", + "sender_display_name": "", + "bracketed_verified_sender": "Bob Smith", + "bracketed_room_name": "Bob's Emporium of Messages", + "to": "person@test.test", + "token": "a_token", + "ephemeral_private_key": "mystery_key", + "web_client_location": "https://matrix.org", + } templateFile = self.sydent.get_branded_template( "matrix-org", @@ -96,7 +108,7 @@ def test_jinja_matrix_invite(self): ) with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") @@ -110,21 +122,36 @@ def test_jinja_matrix_invite(self): # test safe values are not escaped self.assertIn("", email_contents) - #test our link is as expected - expected_url = 'https://matrix.org/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' + # test our link is as expected + expected_url = ( + "https://matrix.org/#/room/" + + urllib.parse.quote("!something:example.org") + + "?email=" + + urllib.parse.quote("test@test.com") + + "&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D" + + urllib.parse.quote("a_token") + + "%26private_key%3D" + + urllib.parse.quote("mystery_key") + + "&room_name=" + + urllib.parse.quote("Bob's Emporium of Messages") + + "&room_avatar_url=" + + urllib.parse.quote("mxc://example.org/s0meM3dia") + + "&inviter_name=" + + urllib.parse.quote("") + + "&guest_access_token=&guest_user_id=" + ) text = email_contents.splitlines() link = text[22] self.assertEqual(link, expected_url) - def test_jinja_matrix_verification(self): substitutions = { - "address": "foo@example.com", - "medium": "email", - "to" : "person@test.test", - "token": "<>", - "link" : "https://link_test.com" - } + "address": "foo@example.com", + "medium": "email", + "to": "person@test.test", + "token": "<>", + "link": "https://link_test.com", + } templateFile = self.sydent.get_branded_template( "matrix-org", @@ -133,7 +160,7 @@ def test_jinja_matrix_verification(self): ) with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") @@ -147,15 +174,13 @@ def test_jinja_matrix_verification(self): # test safe values are not escaped self.assertIn("<>", email_contents) - def test_jinja_vector_verification(self): substitutions = { - "address": "foo@example.com", - "medium": "email", - "to" : "person@test.test", - "token": "<>", - "link" : "https://link_test.com" - } + "address": "foo@example.com", + "medium": "email", + "to": "person@test.test", + "link": "https://link_test.com", + } templateFile = self.sydent.get_branded_template( "vector-im", @@ -164,7 +189,7 @@ def test_jinja_vector_verification(self): ) with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) + sendEmail(self.sydent, templateFile, "test@test.com", substitutions) smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") @@ -172,10 +197,12 @@ def test_jinja_vector_verification(self): # test url input is encoded self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) - # test all output is as expected - path = os.path.join(self.sydent.cfg.get("general", "templates.path"), 'vector_verification_sample.txt') + path = os.path.join( + self.sydent.cfg.get("general", "templates.path"), + "vector_verification_sample.txt", + ) - with open(path, 'r') as file: + with open(path, "r") as file: expected_text = file.read() # remove headers as they are variable @@ -184,4 +211,6 @@ def test_jinja_vector_verification(self): # remove multipart headers as they are variable del email_contents[40] del email_contents[156] + + # test all ouput is as expected self.assertEqual("\n".join(email_contents), expected_text) diff --git a/tests/test_templates.py b/tests/test_templates.py deleted file mode 100644 index d37afc83..00000000 --- a/tests/test_templates.py +++ /dev/null @@ -1,187 +0,0 @@ -import os.path -from unittest.mock import patch - -from twisted.trial import unittest - -from sydent.util.emailutils import sendEmail -from tests.utils import make_sydent, make_request - -import urllib - - -class TestTemplate(unittest.TestCase): - def setUp(self): - # Create a new sydent - config = { - "general": { - "templates.path": os.path.join( - os.path.dirname(os.path.dirname(__file__)), "res" - ), - }, - } - self.sydent = make_sydent(test_config=config) - - - def test_jinja_vector_invite(self): - substitutions = { - "address": "foo@example.com", - "medium": "email", - "room_alias": "#somewhere:exmaple.org", - "room_avatar_url": "mxc://example.org/s0meM3dia", - "room_id": "!something:example.org", - "room_name": "Bob's Emporium of Messages", - "sender": "@bob:example.com", - "sender_avatar_url": "mxc://example.org/an0th3rM3dia", - "sender_display_name": "", - "bracketed_verified_sender" : "Bob Smith", - "bracketed_room_name": "Bob's Emporium of Messages", - "web_client_location" : "element", - "to" : "person@test.test", - "token": "a_token", - "ephemeral_private_key" : "mystery_key", - "web_client_location" : "https://app.element.io" - } - - templateFile = self.sydent.get_branded_template( - "vector-im", - "invite_template.eml.j2", - ("email", "email.invite_template"), - ) - - with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) - - smtp = smtplib.SMTP.return_value - email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - - # test url input is encoded - self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) - - # test html input is escaped - self.assertIn("Bob's Emporium of Messages", email_contents) - - # test safe values are not escaped - self.assertIn("", email_contents) - - #test our link is as expected - expected_url = 'https://app.element.io/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fvector.im%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' - text = email_contents.splitlines() - link = text[19] - self.assertEqual(link, expected_url) - - def test_jinja_matrix_invite(self): - substitutions = { - "address": "foo@example.com", - "medium": "email", - "room_alias": "#somewhere:exmaple.org", - "room_avatar_url": "mxc://example.org/s0meM3dia", - "room_id": "!something:example.org", - "room_name": "Bob's Emporium of Messages", - "sender": "@bob:example.com", - "sender_avatar_url": "mxc://example.org/an0th3rM3dia", - "sender_display_name": "", - "bracketed_verified_sender" : "Bob Smith", - "bracketed_room_name": "Bob's Emporium of Messages", - "web_client_location" : "element", - "to" : "person@test.test", - "token": "a_token", - "ephemeral_private_key" : "mystery_key", - "web_client_location" : "https://matrix.org" - } - - templateFile = self.sydent.get_branded_template( - "matrix-org", - "invite_template.eml.j2", - ("email", "email.invite_template"), - ) - - with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) - - smtp = smtplib.SMTP.return_value - email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - - # test url input is encoded - self.assertIn(urllib.parse.quote("mxc://example.org/s0meM3dia"), email_contents) - - # test html input is escaped - self.assertIn("Bob's Emporium of Messages", email_contents) - - # test safe values are not escaped - self.assertIn("", email_contents) - - #test our link is as expected - expected_url = 'https://matrix.org/#/room/'+urllib.parse.quote('!something:example.org')+'?email='+urllib.parse.quote('test@test.com')+'&signurl=https%3A%2F%2Fmatrix.org%2F_matrix%2Fidentity%2Fapi%2Fv1%2Fsign-ed25519%3Ftoken%3D'+urllib.parse.quote('a_token')+'%26private_key%3D'+urllib.parse.quote('mystery_key')+'&room_name='+urllib.parse.quote("Bob's Emporium of Messages")+'&room_avatar_url='+urllib.parse.quote('mxc://example.org/s0meM3dia')+'&inviter_name='+urllib.parse.quote('')+'&guest_access_token=&guest_user_id=' - text = email_contents.splitlines() - link = text[22] - self.assertEqual(link, expected_url) - - - def test_jinja_matrix_verification(self): - substitutions = { - "address": "foo@example.com", - "medium": "email", - "to" : "person@test.test", - "token": "<>", - "link" : "https://link_test.com" - } - - templateFile = self.sydent.get_branded_template( - "matrix-org", - "verification_template.eml.j2", - ("email", "email.verification_template"), - ) - - with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) - - smtp = smtplib.SMTP.return_value - email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - - # test url input is encoded - self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) - - # test html input is escaped - self.assertIn("<<token>>", email_contents) - - # test safe values are not escaped - self.assertIn("<>", email_contents) - - - def test_jinja_vector_verification(self): - substitutions = { - "address": "foo@example.com", - "medium": "email", - "to" : "person@test.test", - "token": "<>", - "link" : "https://link_test.com" - } - - templateFile = self.sydent.get_branded_template( - "vector-im", - "verification_template.eml.j2", - ("email", "email.verification_template"), - ) - - with patch("sydent.util.emailutils.smtplib") as smtplib: - sendEmail(self.sydent, templateFile, 'test@test.com', substitutions) - - smtp = smtplib.SMTP.return_value - email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - - # test url input is encoded - self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) - - # test all output is as expected - path = os.path.join(self.sydent.cfg.get("general", "templates.path"), 'vector_verification_sample.txt') - - with open(path, 'r') as file: - expected_text = file.read() - - # remove headers as they are variable - email_contents = email_contents.splitlines()[13:] - - # remove multipart headers as they are variable - del email_contents[40] - del email_contents[156] - self.assertEqual("\n".join(email_contents), expected_text) From 9880e0c1cc2d16cedeb8088d31eecbc679973eed Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 18 Aug 2021 10:45:14 -0700 Subject: [PATCH 31/38] update sydent.py --- sydent/sydent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sydent/sydent.py b/sydent/sydent.py index ee629b76..7ee96078 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -23,7 +23,7 @@ from typing import Set import twisted.internet.reactor -from jinja2 import Environment, FileSystemLoader, select_autoescape +from jinja2 import Environment, FileSystemLoader from twisted.internet import address, task from twisted.python import log @@ -297,7 +297,7 @@ def __init__( ) self.template_environment = Environment( loader=FileSystemLoader(self.cfg.get("general", "templates.path")), - autoescape=select_autoescape(default=True), + autoescape=True, ) # See if a pepper already exists in the database From 5379f85668df84ea49feee79a28d3a0b237817b8 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 19 Aug 2021 10:57:20 -0700 Subject: [PATCH 32/38] requested changes + lints --- res/matrix-org/verification_template.eml.j2 | 6 +++--- res/vector-im/verification_template.eml.j2 | 4 ++-- res/vector_verification_sample.txt | 7 +++++-- tests/test_jinja_templates.py | 22 ++++++++------------- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/res/matrix-org/verification_template.eml.j2 b/res/matrix-org/verification_template.eml.j2 index fc3e5240..66973247 100644 --- a/res/matrix-org/verification_template.eml.j2 +++ b/res/matrix-org/verification_template.eml.j2 @@ -17,7 +17,7 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -{{ link|urlencode }} +{{ link|safe}} If your client requires a code, the code is {{ token|safe }} @@ -59,11 +59,11 @@ body { identity server. If this was you who made this request, you may use the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

...or copy this link into your web browser:

-

{{ link|urlencode }}

+

{{ link }}

If your client requires a code, the code is {{ token }}

diff --git a/res/vector-im/verification_template.eml.j2 b/res/vector-im/verification_template.eml.j2 index bb20ff2e..66f15396 100644 --- a/res/vector-im/verification_template.eml.j2 +++ b/res/vector-im/verification_template.eml.j2 @@ -19,7 +19,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -{{ link|urlencode }} +{{ link|safe }} Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. @@ -125,7 +125,7 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

Please note that Element requires diff --git a/res/vector_verification_sample.txt b/res/vector_verification_sample.txt index 34fdbc94..e48dce85 100644 --- a/res/vector_verification_sample.txt +++ b/res/vector_verification_sample.txt @@ -6,7 +6,7 @@ distributed and secure shared workspace for the web that's built on Matrix. If it was really you who made this request, you can click on the following link to complete the verification of your email address: -https%3A//link_test.com +https://link_test.com Please note that you will need to use Chrome, Firefox or Safari on the web, or iOS or Android on mobile. @@ -38,6 +38,7 @@ decentralized communication delivering a community of users, bridged networks, integrated bots and applications plus full end-to-end encryption. To learn more about Matrix visit https://matrix.org. +--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Content-Type: text/html; charset=UTF-8 Content-Disposition: inline @@ -111,7 +112,7 @@ pre, code {

If it was really you who made this request, you can click on the following link to complete the verification of your email address:

-

Complete email verification

+

Complete email verification

Please note that Element requires @@ -153,3 +154,5 @@ pre, code { + +--aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-- \ No newline at end of file diff --git a/tests/test_jinja_templates.py b/tests/test_jinja_templates.py index c2bcc590..a95a24e1 100644 --- a/tests/test_jinja_templates.py +++ b/tests/test_jinja_templates.py @@ -1,6 +1,6 @@ import os.path import urllib -from unittest.mock import patch +from unittest.mock import Mock, patch from twisted.trial import unittest @@ -165,15 +165,16 @@ def test_jinja_matrix_verification(self): smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - # test url input is encoded - self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) - # test html input is escaped self.assertIn("<<token>>", email_contents) # test safe values are not escaped self.assertIn("<>", email_contents) + @patch( + "sydent.util.emailutils.generateAlphanumericTokenOfLength", + Mock(return_value="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ) def test_jinja_vector_verification(self): substitutions = { "address": "foo@example.com", @@ -194,9 +195,6 @@ def test_jinja_vector_verification(self): smtp = smtplib.SMTP.return_value email_contents = smtp.sendmail.call_args[0][2].decode("utf-8") - # test url input is encoded - self.assertIn(urllib.parse.quote("https://link_test.com"), email_contents) - path = os.path.join( self.sydent.cfg.get("general", "templates.path"), "vector_verification_sample.txt", @@ -205,12 +203,8 @@ def test_jinja_vector_verification(self): with open(path, "r") as file: expected_text = file.read() - # remove headers as they are variable - email_contents = email_contents.splitlines()[13:] - - # remove multipart headers as they are variable - del email_contents[40] - del email_contents[156] + # remove the email headers as they are variable + email_contents = email_contents[email_contents.index("Hello") :] # test all ouput is as expected - self.assertEqual("\n".join(email_contents), expected_text) + self.assertEqual(email_contents, expected_text) From 611ee4a392158b5d8dc52628270212f741cc5f01 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 23 Aug 2021 09:36:44 -0700 Subject: [PATCH 33/38] update templates --- res/matrix-org/verification_template.eml.j2 | 2 +- res/vector-im/invite_template.eml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/res/matrix-org/verification_template.eml.j2 b/res/matrix-org/verification_template.eml.j2 index 66973247..fffbb87e 100644 --- a/res/matrix-org/verification_template.eml.j2 +++ b/res/matrix-org/verification_template.eml.j2 @@ -17,7 +17,7 @@ We have received a request to use this email address with a matrix.org identity server. If this was you who made this request, you may use the following link to complete the verification of your email address: -{{ link|safe}} +{{ link|safe }} If your client requires a code, the code is {{ token|safe }} diff --git a/res/vector-im/invite_template.eml b/res/vector-im/invite_template.eml index 1ca18025..8cd8d0dc 100644 --- a/res/vector-im/invite_template.eml +++ b/res/vector-im/invite_template.eml @@ -17,10 +17,8 @@ Hi, %(bracketed_room_name)son Element. To join the conversation please follow the link below. - %(web_client_location)s/#/room/%(room_id_forurl)s?email=%(to_forurl)s&signurl=https%%3A%%2F%%2Fvector.im%%2F_matrix%%2Fidentity%%2Fapi%%2Fv1%%2Fsign-ed25519%%3Ftoken%%3D%(token)s%%26private_key%%3D%(ephemeral_private_key)s&room_name=%(room_name_forurl)s&room_avatar_url=%(room_avatar_url_forurl)s&inviter_name=%(sender_display_name_forurl)s&guest_access_token=%(guest_access_token_forurl)s&guest_user_id=%(guest_user_id_forurl)s - Element is an open source collaboration app built on the Matrix.org open standard for interoperable communication: supporting group chat, file transfer, voice and video calling, integrations to other apps, bridges From 0baa33c033b4a427750d31228ef78917cf943eb3 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 23 Aug 2021 09:37:57 -0700 Subject: [PATCH 34/38] requested changes + add licence --- sydent/sydent.py | 2 +- sydent/util/emailutils.py | 33 ++++++++++++++------------------- tests/test_jinja_templates.py | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/sydent/sydent.py b/sydent/sydent.py index 7ee96078..0f27657a 100644 --- a/sydent/sydent.py +++ b/sydent/sydent.py @@ -482,7 +482,7 @@ def get_branded_template(self, brand, template_name, deprecated_template_name): if os.path.exists( os.path.join(root_template_path, brand, template_name + ".j2") ): - return os.path.join(root_template_path, brand, template_name + ".j2") + return os.path.join(brand, template_name + ".j2") else: return os.path.join(root_template_path, brand, template_name) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 1802697d..def4d272 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -65,26 +65,21 @@ def sendEmail( allSubstitutions = {} - for k, v in substitutions.items(): - allSubstitutions[k] = v - if not templateFile.endswith(".j2"): - allSubstitutions[k + "_forhtml"] = escape(v) - allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) - - # We add randomize the multipart boundary to stop user input from - # conflicting with it. - allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) - - # extract branded template name from templateFile - file_parts = templateFile.split("/") - template_name = file_parts[-2] + "/" + file_parts[-1] - # use jinja for rendering if jinja templates are present - with open(templateFile) as template_file: - if templateFile.endswith(".j2"): - template = sydent.template_environment.get_template(template_name) - mailString = template.render(allSubstitutions) - else: + if templateFile.endswith(".j2"): + for k, v in substitutions.items(): + allSubstitutions[k] = v + # We add randomize the multipart boundary to stop user input from + # conflicting with it. + allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) + template = sydent.template_environment.get_template(templateFile) + mailString = template.render(allSubstitutions) + else: + with open(templateFile) as template_file: + for k, v in substitutions.items(): + allSubstitutions[k + "_forhtml"] = escape(v) + allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) + allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) mailString = template_file.read() % allSubstitutions parsedFrom = email.utils.parseaddr(mailFrom)[1] diff --git a/tests/test_jinja_templates.py b/tests/test_jinja_templates.py index a95a24e1..06ef612b 100644 --- a/tests/test_jinja_templates.py +++ b/tests/test_jinja_templates.py @@ -1,3 +1,17 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os.path import urllib from unittest.mock import Mock, patch From 4010fd5532766640b1520cb36ba50d3d4710dca4 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 23 Aug 2021 09:39:36 -0700 Subject: [PATCH 35/38] lints --- sydent/util/emailutils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index def4d272..5e96cb7c 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -79,7 +79,9 @@ def sendEmail( for k, v in substitutions.items(): allSubstitutions[k + "_forhtml"] = escape(v) allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) - allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) + allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength( + 32 + ) mailString = template_file.read() % allSubstitutions parsedFrom = email.utils.parseaddr(mailFrom)[1] From a9dcbcf78f2b88d52eda358f4683d2b61802b828 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 23 Aug 2021 09:43:38 -0700 Subject: [PATCH 36/38] slight refactor --- sydent/util/emailutils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 5e96cb7c..91344669 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -67,13 +67,11 @@ def sendEmail( # use jinja for rendering if jinja templates are present if templateFile.endswith(".j2"): - for k, v in substitutions.items(): - allSubstitutions[k] = v # We add randomize the multipart boundary to stop user input from # conflicting with it. - allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) + substitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) template = sydent.template_environment.get_template(templateFile) - mailString = template.render(allSubstitutions) + mailString = template.render(substitutions) else: with open(templateFile) as template_file: for k, v in substitutions.items(): From d4d048673d92171530eb659b8dc9af3ea74d9ebb Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 24 Aug 2021 09:33:48 -0700 Subject: [PATCH 37/38] requested change + lints --- sydent/util/emailutils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 91344669..f2e63cb5 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -63,8 +63,6 @@ def sendEmail( } ) - allSubstitutions = {} - # use jinja for rendering if jinja templates are present if templateFile.endswith(".j2"): # We add randomize the multipart boundary to stop user input from @@ -73,13 +71,12 @@ def sendEmail( template = sydent.template_environment.get_template(templateFile) mailString = template.render(substitutions) else: + allSubstitutions = {} + for k, v in substitutions.items(): + allSubstitutions[k + "_forhtml"] = escape(v) + allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) + allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32) with open(templateFile) as template_file: - for k, v in substitutions.items(): - allSubstitutions[k + "_forhtml"] = escape(v) - allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) - allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength( - 32 - ) mailString = template_file.read() % allSubstitutions parsedFrom = email.utils.parseaddr(mailFrom)[1] From b0599a7ea4ab8d6ff0e31516bca47cf1bd39eeaa Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 24 Aug 2021 10:04:23 -0700 Subject: [PATCH 38/38] fix error in redering emails --- sydent/util/emailutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index f2e63cb5..6e532dc3 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -73,6 +73,7 @@ def sendEmail( else: allSubstitutions = {} for k, v in substitutions.items(): + allSubstitutions[k] = v allSubstitutions[k + "_forhtml"] = escape(v) allSubstitutions[k + "_forurl"] = urllib.parse.quote(v) allSubstitutions["multipart_boundary"] = generateAlphanumericTokenOfLength(32)