From d28b9df393c87777a902634254d464a192261600 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 1 Jun 2021 15:44:18 -0700 Subject: [PATCH 01/14] reset author --- sydent/db/accounts.py | 16 ++++++---- sydent/db/hashing_metadata.py | 16 ++++++++-- sydent/db/invite_tokens.py | 18 +++++++----- sydent/db/peers.py | 12 ++++---- sydent/db/terms.py | 6 ++-- sydent/db/threepid_associations.py | 47 ++++++++++++++++++++++-------- sydent/db/valsession.py | 29 ++++++++++++------ 7 files changed, 101 insertions(+), 43 deletions(-) diff --git a/sydent/db/accounts.py b/sydent/db/accounts.py index 52b0cb85..dbed90e4 100644 --- a/sydent/db/accounts.py +++ b/sydent/db/accounts.py @@ -17,12 +17,14 @@ from sydent.users.accounts import Account +from typing import Union + class AccountStore(object): def __init__(self, sydent): self.sydent = sydent - def getAccountByToken(self, token): + def getAccountByToken(self, token: str) -> Union[Account, None]: """ Select the account matching the given token, if any. @@ -45,7 +47,9 @@ def getAccountByToken(self, token): return Account(*row) - def storeAccount(self, user_id, creation_ts, consent_version): + def storeAccount( + self, user_id: str, creation_ts: int, consent_version: Union[bytes, None] + ) -> None: """ Stores an account for the given user ID. @@ -65,7 +69,9 @@ def storeAccount(self, user_id, creation_ts, consent_version): ) self.sydent.db.commit() - def setConsentVersion(self, user_id, consent_version): + def setConsentVersion( + self, user_id: bytes, consent_version: Union[str, None] + ) -> None: """ Saves that the given user has agreed to all of the terms in the document of the given version. @@ -82,7 +88,7 @@ def setConsentVersion(self, user_id, consent_version): ) self.sydent.db.commit() - def addToken(self, user_id, token): + def addToken(self, user_id: str, token: str) -> None: """ Stores the authentication token for a given user. @@ -98,7 +104,7 @@ def addToken(self, user_id, token): ) self.sydent.db.commit() - def delToken(self, token): + def delToken(self, token: str) -> int: """ Deletes an authentication token from the database. diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index 111224f4..f670cef3 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -17,12 +17,14 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py +from typing import Union, Callable + class HashingMetadataStore: def __init__(self, sydent): self.sydent = sydent - def get_lookup_pepper(self): + def get_lookup_pepper(self) -> Union[str, None]: """Return the value of the current lookup pepper from the db :return: A pepper if it exists in the database, or None if one does @@ -44,7 +46,9 @@ def get_lookup_pepper(self): return pepper - def store_lookup_pepper(self, hashing_function, pepper): + def store_lookup_pepper( + self, hashing_function: Callable[[str], str], pepper: str + ) -> None: """Stores a new lookup pepper in the hashing_metadata db table and rehashes all 3PIDs :param hashing_function: A function with single input and output strings @@ -74,7 +78,13 @@ def store_lookup_pepper(self, hashing_function, pepper): # Commit the queued db transactions so that adding a new pepper and hashing is atomic self.sydent.db.commit() - def _rehash_threepids(self, cur, hashing_function, pepper, table): + def _rehash_threepids( + self, + cur: object, + hashing_function: Callable[[str], str], + pepper: str, + table: str, + ) -> None: """Rehash 3PIDs of a given table using a given hashing_function and pepper A database cursor `cur` must be passed to this function. After this function completes, diff --git a/sydent/db/invite_tokens.py b/sydent/db/invite_tokens.py index 8b92b677..c968883c 100644 --- a/sydent/db/invite_tokens.py +++ b/sydent/db/invite_tokens.py @@ -15,12 +15,16 @@ # limitations under the License. import time +from typing import Union, List, Dict + class JoinTokenStore(object): def __init__(self, sydent): self.sydent = sydent - def storeToken(self, medium, address, roomId, sender, token): + def storeToken( + self, medium: str, address: str, roomId: str, sender: str, token: str + ) -> None: """ Store a new invite token and its metadata. @@ -45,7 +49,7 @@ def storeToken(self, medium, address, roomId, sender, token): ) self.sydent.db.commit() - def getTokens(self, medium, address): + def getTokens(self, medium: str, address: str) -> List[Dict[str, str]]: """ Retrieves the pending invites tokens for this 3PID that haven't been delivered yet. @@ -100,7 +104,7 @@ def getTokens(self, medium, address): return ret - def markTokensAsSent(self, medium, address): + def markTokensAsSent(self, medium: str, address: str) -> None: """ Updates the invite tokens associated with a given 3PID to mark them as delivered to a homeserver so they're not delivered again in the future. @@ -122,7 +126,7 @@ def markTokensAsSent(self, medium, address): ) self.sydent.db.commit() - def storeEphemeralPublicKey(self, publicKey): + def storeEphemeralPublicKey(self, publicKey: str) -> None: """ Saves the provided ephemeral public key. @@ -138,7 +142,7 @@ def storeEphemeralPublicKey(self, publicKey): ) self.sydent.db.commit() - def validateEphemeralPublicKey(self, publicKey): + def validateEphemeralPublicKey(self, publicKey: str) -> bool: """ Checks if an ephemeral public key is valid, and, if it is, updates its verification count. @@ -159,7 +163,7 @@ def validateEphemeralPublicKey(self, publicKey): self.sydent.db.commit() return cur.rowcount > 0 - def getSenderForToken(self, token): + def getSenderForToken(self, token: str) -> Union[str, None]: """ Retrieves the MXID of the user that sent the invite the provided token is for. @@ -177,7 +181,7 @@ def getSenderForToken(self, token): return rows[0][0] return None - def deleteTokens(self, medium, address): + def deleteTokens(self, medium: str, address: str) -> None: """ Deletes every token for a given 3PID. diff --git a/sydent/db/peers.py b/sydent/db/peers.py index 3c796ad6..4aade6e1 100644 --- a/sydent/db/peers.py +++ b/sydent/db/peers.py @@ -17,12 +17,14 @@ from sydent.replication.peer import RemotePeer +from typing import Union, List, Dict + class PeerStore: def __init__(self, sydent): self.sydent = sydent - def getPeerByName(self, name): + def getPeerByName(self, name: str) -> Union[RemotePeer, None]: """ Retrieves a remote peer using it's server name. @@ -57,7 +59,7 @@ def getPeerByName(self, name): return p - def getAllPeers(self): + def getAllPeers(self) -> List[RemotePeer]: """ Retrieve all of the remote peers from the database. @@ -75,7 +77,7 @@ def getAllPeers(self): peername = None port = None lastSentVer = None - pubkeys = {} + pubkeys: Dict[str, str] = {} for row in res.fetchall(): if row[0] != peername: @@ -95,8 +97,8 @@ def getAllPeers(self): return peers def setLastSentVersionAndPokeSucceeded( - self, peerName, lastSentVersion, lastPokeSucceeded - ): + self, peerName: str, lastSentVersion: int, lastPokeSucceeded: int + ) -> None: """ Sets the ID of the last association sent to a given peer and the time of the last successful request sent to that peer. diff --git a/sydent/db/terms.py b/sydent/db/terms.py index 6e27715a..974f74dc 100644 --- a/sydent/db/terms.py +++ b/sydent/db/terms.py @@ -14,12 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + class TermsStore(object): def __init__(self, sydent): self.sydent = sydent - def getAgreedUrls(self, user_id): + def getAgreedUrls(self, user_id: str) -> List[str]: """ Retrieves the URLs of the terms the given user has agreed to. @@ -45,7 +47,7 @@ def getAgreedUrls(self, user_id): return urls - def addAgreedUrls(self, user_id, urls): + def addAgreedUrls(self, user_id: str, urls: List[str]) -> None: """ Saves that the given user has accepted the terms at the given URLs. diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index ee39b692..64ee4c29 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -19,7 +19,15 @@ from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer +<<<<<<< HEAD from sydent.util import time_msec +======= + +import logging + +from typing import Tuple, Union, List, Dict, Any, Sequence + +>>>>>>> reset author logger = logging.getLogger(__name__) @@ -28,7 +36,7 @@ class LocalAssociationStore: def __init__(self, sydent): self.sydent = sydent - def addOrUpdateAssociation(self, assoc): + def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: """ Updates an association, or creates one if none exists with these parameters. @@ -54,7 +62,9 @@ def addOrUpdateAssociation(self, assoc): ) self.sydent.db.commit() - def getAssociationsAfterId(self, afterId, limit=None): + def getAssociationsAfterId( + self, afterId: int, limit: Union[int, None] = None + ) -> Tuple[Dict[int, ThreepidAssociation], Union[int, None]]: """ Retrieves every association after the given ID. @@ -97,7 +107,9 @@ def getAssociationsAfterId(self, afterId, limit=None): return assocs, maxId - def getSignedAssociationsAfterId(self, afterId, limit=None): + def getSignedAssociationsAfterId( + self, afterId: int, limit: Union[int, None] = None + ) -> Tuple[Dict[int, Dict[str, Any]], Union[int, None]]: """Get associations after a given ID, and sign them before returning :param afterId: The ID to return results after (not inclusive) @@ -124,7 +136,7 @@ def getSignedAssociationsAfterId(self, afterId, limit=None): return assocs, maxId - def removeAssociation(self, threepid, mxid): + def removeAssociation(self, threepid: Dict[str, str], mxid: str) -> None: """ Delete the association between a 3PID and a MXID, if it exists. If the association doesn't exist, log and do nothing. @@ -179,7 +191,9 @@ class GlobalAssociationStore: def __init__(self, sydent): self.sydent = sydent - def signedAssociationStringForThreepid(self, medium, address): + def signedAssociationStringForThreepid( + self, medium: str, address: str + ) -> Union[str, None]: """ Retrieve the JSON for the signed association matching the provided 3PID, if one exists. @@ -214,7 +228,7 @@ def signedAssociationStringForThreepid(self, medium, address): return sgAssocStr - def getMxid(self, medium, address): + def getMxid(self, medium: str, address: str) -> Union[str, None]: """ Retrieves the MXID associated with a 3PID. @@ -241,7 +255,9 @@ def getMxid(self, medium, address): return row[0] - def getMxids(self, threepid_tuples): + def getMxids( + self, threepid_tuples: List[Tuple[str, str, str]] + ) -> List[Tuple[str, str, str]]: """Given a list of threepid_tuples, return the same list but with mxids appended to each tuple for which a match was found in the database for. Output is ordered by medium, address, timestamp DESC @@ -281,7 +297,7 @@ def getMxids(self, threepid_tuples): ) results = [] - current = () + current = (Any, Any) for row in res.fetchall(): # only use the most recent entry for each # threepid (they're sorted by ts) @@ -295,7 +311,14 @@ def getMxids(self, threepid_tuples): return results - def addAssociation(self, assoc, rawSgAssoc, originServer, originId, commit=True): + def addAssociation( + self, + assoc: ThreepidAssociation, + rawSgAssoc: Dict[str, Any], + originServer: str, + originId: int, + commit: bool = True, + ) -> None: """ Saves an association received through either a replication push or a local push. @@ -333,7 +356,7 @@ def addAssociation(self, assoc, rawSgAssoc, originServer, originId, commit=True) if commit: self.sydent.db.commit() - def lastIdFromServer(self, server): + def lastIdFromServer(self, server: str) -> Union[int, None]: """ Retrieves the ID of the last association received from the given peer. @@ -357,7 +380,7 @@ def lastIdFromServer(self, server): return row[0] - def removeAssociation(self, medium, address): + def removeAssociation(self, medium: str, address: str) -> None: """ Removes any association stored for the provided 3PID. @@ -380,7 +403,7 @@ def removeAssociation(self, medium, address): ) self.sydent.db.commit() - def retrieveMxidsForHashes(self, addresses): + def retrieveMxidsForHashes(self, addresses: List[str]) -> Dict[str, str]: """Returns a mapping from hash: mxid from a list of given lookup_hash values :param addresses: An array of lookup_hash values to check against the db diff --git a/sydent/db/valsession.py b/sydent/db/valsession.py index f52d777b..e944c432 100644 --- a/sydent/db/valsession.py +++ b/sydent/db/valsession.py @@ -27,13 +27,17 @@ ValidationSession, ) +from typing import Union + class ThreePidValSessionStore: def __init__(self, syd): self.sydent = syd self.random = SystemRandom() - def getOrCreateTokenSession(self, medium, address, clientSecret): + def getOrCreateTokenSession( + self, medium: str, address: str, clientSecret: str + ) -> ValidationSession: """ Retrieves the validation session for a given medium, address and client secret, or creates one if none was found. @@ -82,7 +86,14 @@ def getOrCreateTokenSession(self, medium, address, clientSecret): ) return s - def addValSession(self, medium, address, clientSecret, mtime, commit=True): + def addValSession( + self, + medium: str, + address: str, + clientSecret: str, + mtime: int, + commit: bool = True, + ) -> int: """ Creates a validation session with the given parameters. @@ -117,7 +128,7 @@ def addValSession(self, medium, address, clientSecret, mtime, commit=True): self.sydent.db.commit() return sid - def setSendAttemptNumber(self, sid, attemptNo): + def setSendAttemptNumber(self, sid: str, attemptNo: int) -> None: """ Updates the send attempt number for the session with the given ID. @@ -134,7 +145,7 @@ def setSendAttemptNumber(self, sid, attemptNo): ) self.sydent.db.commit() - def setValidated(self, sid, validated): + def setValidated(self, sid: str, validated: bool) -> None: """ Updates a session to set the validated flag to the given value. @@ -151,7 +162,7 @@ def setValidated(self, sid, validated): ) self.sydent.db.commit() - def setMtime(self, sid, mtime): + def setMtime(self, sid: str, mtime: int) -> None: """ Set the time of the last send attempt for the session with the given ID @@ -168,7 +179,7 @@ def setMtime(self, sid, mtime): ) self.sydent.db.commit() - def getSessionById(self, sid): + def getSessionById(self, sid: str) -> Union[ValidationSession, None]: """ Retrieves the session matching the given sid. @@ -195,7 +206,7 @@ def getSessionById(self, sid): row[0], row[1], row[2], row[3], row[4], row[5], None, None ) - def getTokenSessionById(self, sid): + def getTokenSessionById(self, sid: str) -> Union[ValidationSession, None]: """ Retrieves a validation session using the session's ID. @@ -223,7 +234,7 @@ def getTokenSessionById(self, sid): return None - def getValidatedSession(self, sid, clientSecret): + def getValidatedSession(self, sid: str, clientSecret: str) -> ValidationSession: """ Retrieve a validated and still-valid session whose client secret matches the one passed in. @@ -260,7 +271,7 @@ def getValidatedSession(self, sid, clientSecret): return s - def deleteOldSessions(self): + def deleteOldSessions(self) -> None: """Delete old threepid validation sessions that are long expired.""" cur = self.sydent.db.cursor() From 3359fa3a0aa7d8a7550537479d6572efdec2715e Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Wed, 2 Jun 2021 08:41:37 -0700 Subject: [PATCH 02/14] resolved sqlite3 name for mypy --- sydent/db/hashing_metadata.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index f670cef3..25a3866e 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -17,7 +17,10 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py -from typing import Union, Callable +from typing import Union, Callable, TYPE_CHECKING + +if TYPE_CHECKING: + import sqlite3 class HashingMetadataStore: @@ -80,7 +83,7 @@ def store_lookup_pepper( def _rehash_threepids( self, - cur: object, + cur: "sqlite3.Cursor", hashing_function: Callable[[str], str], pepper: str, table: str, From 544abc06525d71e29d56ccb1d23576650f164d8a Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 3 Jun 2021 09:58:31 -0700 Subject: [PATCH 03/14] add type hints --- sydent/db/threepid_associations.py | 2 +- sydent/hs_federation/verifier.py | 28 +++++++++++++++++++++++----- sydent/http/auth.py | 16 ++++++++++++++-- sydent/http/httpclient.py | 15 +++++++++++++-- sydent/replication/peer.py | 28 +++++++++++++++++++++------- sydent/sms/openmarket.py | 6 ++++-- sydent/terms/terms.py | 14 ++++++++------ sydent/threepid/bind.py | 2 +- 8 files changed, 85 insertions(+), 26 deletions(-) diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index 64ee4c29..1ef41df0 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -403,7 +403,7 @@ def removeAssociation(self, medium: str, address: str) -> None: ) self.sydent.db.commit() - def retrieveMxidsForHashes(self, addresses: List[str]) -> Dict[str, str]: + def retrieveMxidsForHashes(self, addresses: List[Any]) -> Dict[str, str]: """Returns a mapping from hash: mxid from a list of given lookup_hash values :param addresses: An array of lookup_hash values to check against the db diff --git a/sydent/hs_federation/verifier.py b/sydent/hs_federation/verifier.py index 31481a66..ca80164d 100644 --- a/sydent/hs_federation/verifier.py +++ b/sydent/hs_federation/verifier.py @@ -18,8 +18,13 @@ import logging import time + import signedjson.key import signedjson.sign +from twisted.internet import defer +from unpaddedbase64 import decode_base64 # type: ignore +import signedjson.sign #type: ignore +import signedjson.key #type: ignore from signedjson.sign import SignatureVerifyException from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -27,6 +32,11 @@ from sydent.http.httpclient import FederationHttpClient from sydent.util.stringutils import is_valid_matrix_server_name +from typing import Dict, Union, TYPE_CHECKING, Any, List, Tuple, Generator + +if TYPE_CHECKING: + import twisted + logger = logging.getLogger(__name__) @@ -62,7 +72,9 @@ def __init__(self, sydent): } @defer.inlineCallbacks - def _getKeysForServer(self, server_name): + def _getKeysForServer( + self, server_name: str + ) -> Generator: """Get the signing key data from a homeserver. :param server_name: The name of the server to request the keys from. @@ -105,7 +117,11 @@ def _getKeysForServer(self, server_name): defer.returnValue(result["verify_keys"]) @defer.inlineCallbacks - def verifyServerSignedJson(self, signed_json, acceptable_server_names=None): + def verifyServerSignedJson( + self, + signed_json: Dict[str, Any], + acceptable_server_names: Union[List[str], None] = None, + ) -> Generator: """Given a signed json object, try to verify any one of the signatures on it @@ -162,7 +178,9 @@ def verifyServerSignedJson(self, signed_json, acceptable_server_names=None): raise SignatureVerifyException("No matching signature found") @defer.inlineCallbacks - def authenticate_request(self, request, content): + def authenticate_request( + self, request: twisted.web.server.Request, content: Union[bytes, None] + ) -> Generator: """Authenticates a Matrix federation request based on the X-Matrix header XXX: Copied largely from synapse @@ -186,7 +204,7 @@ def authenticate_request(self, request, content): origin = None - def parse_auth_header(header_str): + def parse_auth_header(header_str: str) -> Tuple[str, str, str]: """ Extracts a server name, signing key and payload signature from an authentication header. @@ -199,7 +217,7 @@ def parse_auth_header(header_str): """ try: params = header_str.split(u" ")[1].split(u",") - param_dict = dict(kv.split(u"=") for kv in params) + param_dict: Dict[str, str] = dict(kv.split(u"=") for kv in params) def strip_quotes(value): if value.startswith(u'"'): diff --git a/sydent/http/auth.py b/sydent/http/auth.py index fde36bb3..3b15db88 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -21,10 +21,18 @@ from sydent.http.servlets import MatrixRestError, get_args from sydent.terms.terms import get_terms +from typing import Union, TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.db.accounts import Account + import twisted + from sydent.sydent import Sydent + + logger = logging.getLogger(__name__) -def tokenFromRequest(request): +def tokenFromRequest(request: twisted.web.server.Request) -> Optional[str]: """Extract token from header of query parameter. :param request: The request to look for an access token in. @@ -51,7 +59,11 @@ def tokenFromRequest(request): return token -def authV2(sydent, request, requireTermsAgreed=True): +def authV2( + sydent: sydent.sydent.Sydent, + request: twisted.web.server.Request, + requireTermsAgreed: bool = True, +) -> Union[Account, None]: """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py index 54f4f1b7..5445f395 100644 --- a/sydent/http/httpclient.py +++ b/sydent/http/httpclient.py @@ -29,6 +29,11 @@ from sydent.http.matrixfederationagent import MatrixFederationAgent from sydent.util import json_decoder +from typing import Union, Dict, Any, TYPE_CHECKING, Generator + +if TYPE_CHECKING: + import twisted + logger = logging.getLogger(__name__) @@ -36,9 +41,13 @@ class HTTPClient(object): """A base HTTP class that contains methods for making GET and POST HTTP requests. """ + agent: Any # TODO: find actual type of twisted.web.iweb.IAgent + @defer.inlineCallbacks - def get_json(self, uri, max_size=None): + def get_json( + self, uri: str, max_size: Union[int, None] = None + ) -> Generator[Dict[Any, Any], Any, Any]: """Make a GET request to an endpoint returning JSON and parse result :param uri: The URI to make a GET request to. @@ -66,7 +75,9 @@ def get_json(self, uri, max_size=None): defer.returnValue(json_body) @defer.inlineCallbacks - def post_json_get_nothing(self, uri, post_json, opts): + def post_json_get_nothing( + self, uri: str, post_json: Dict[Any, Any], opts: Dict[str, Any] + ) -> Generator[Any, Any, Any]: """Make a POST request to an endpoint returning JSON and parse result :param uri: The URI to make a POST request to. diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py index c969d224..b06b668b 100644 --- a/sydent/replication/peer.py +++ b/sydent/replication/peer.py @@ -33,6 +33,20 @@ from sydent.threepid import threePidAssocFromDict from sydent.util import json_decoder from sydent.util.hash import sha256_and_url_safe_base64 +from unpaddedbase64 import decode_base64 # type: ignore + +import signedjson.sign # type: ignore +import signedjson.key # type: ignore + +import logging +import json +import binascii + +from twisted.internet import defer +from twisted.web.client import readBody + +from collections.abc import Sequence +from typing import Generator logger = logging.getLogger(__name__) @@ -45,7 +59,7 @@ def __init__(self, servername, pubkeys): self.pubkeys = pubkeys self.is_being_pushed_to = False - def pushUpdates(self, sgAssocs): + def pushUpdates(self, sgAssocs: Sequence[tuple]) -> Generator: """ :param sgAssocs: Sequence of (originId, sgAssoc) tuples where originId is the id on the creating server and sgAssoc is the json object of the signed association @@ -69,7 +83,7 @@ def __init__(self, sydent): if self.lastId is None: self.lastId = -1 - def pushUpdates(self, sgAssocs): + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: """ Saves the given associations in the global associations store. Only stores an association if its ID is greater than the last seen ID. @@ -181,7 +195,7 @@ def __init__(self, sydent, server_name, port, pubkeys, lastSentVersion): self.verify_key.alg = SIGNING_KEY_ALGORITHM self.verify_key.version = 0 - def verifySignedAssociation(self, assoc): + def verifySignedAssociation(self, assoc: Dict[Any, Any]) -> None: """Verifies a signature on a signed association. Raises an exception if the signature is incorrect or couldn't be verified. @@ -205,7 +219,7 @@ def verifySignedAssociation(self, assoc): # Verify the JSON signedjson.sign.verify_signed_json(assoc, self.servername, self.verify_key) - def pushUpdates(self, sgAssocs): + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: """ Pushes the given associations to the peer. @@ -233,7 +247,7 @@ def pushUpdates(self, sgAssocs): return updateDeferred - def _pushSuccess(self, result, updateDeferred): + def _pushSuccess(self, result: twisted.web.iweb.IResponse, updateDeferred: twisted.internet.defer.Deferred) -> None: """ Processes a successful push request. If the request resulted in a status code that's not a success, consider it a failure @@ -251,7 +265,7 @@ def _pushSuccess(self, result, updateDeferred): d.addCallback(self._failedPushBodyRead, updateDeferred=updateDeferred) d.addErrback(self._pushFailed, updateDeferred=updateDeferred) - def _failedPushBodyRead(self, body, updateDeferred): + def _failedPushBodyRead(self, body: bytes, updateDeferred: twisted.internet.defer.Deferred) -> None: """ Processes a response body from a failed push request, then calls the error callback of the provided deferred. @@ -266,7 +280,7 @@ def _failedPushBodyRead(self, body, updateDeferred): e.errorDict = errObj updateDeferred.errback(e) - def _pushFailed(self, failure, updateDeferred): + def _pushFailed(self, failure: twisted.python.failure.Failure, updateDeferred: twisted.internet.defer.Deferred) -> None: """ Processes a failed push request, by calling the error callback of the given deferred with it. diff --git a/sydent/sms/openmarket.py b/sydent/sms/openmarket.py index 8af372f1..3f35c413 100644 --- a/sydent/sms/openmarket.py +++ b/sydent/sms/openmarket.py @@ -23,6 +23,8 @@ from sydent.http.httpclient import SimpleHttpClient +from typing import Union, Dict, Generator, Any + logger = logging.getLogger(__name__) @@ -61,7 +63,7 @@ def __init__(self, sydent): self.http_cli = SimpleHttpClient(sydent) @defer.inlineCallbacks - def sendTextSMS(self, body, dest, source=None): + def sendTextSMS(self, body: Dict, dest: str, source: Union[None, Dict[str, str]]=None) -> Generator: """ Sends a text message with the given body to the given MSISDN. @@ -91,7 +93,7 @@ def sendTextSMS(self, body, dest, source=None): password = self.sydent.cfg.get("sms", "password").encode("UTF-8") b64creds = b64encode(b"%s:%s" % (username, password)) - headers = Headers( + headers: Any = Headers( { b"Authorization": [b"Basic " + b64creds], b"Content-Type": [b"application/json"], diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index e7361aad..370f729c 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -17,19 +17,20 @@ import logging import yaml +from typing import Union, Set, List, Dict, Optional logger = logging.getLogger(__name__) class Terms(object): - def __init__(self, yamlObj): + def __init__(self, yamlObj) -> None: """ :param yamlObj: The parsed YAML. :type yamlObj: dict[str, any] or None """ self._rawTerms = yamlObj - def getMasterVersion(self): + def getMasterVersion(self) -> Union[str, None]: """ :return: The global (master) version of the terms, or None if there are no terms of service for this server. @@ -43,7 +44,7 @@ def getMasterVersion(self): return version - def getForClient(self): + def getForClient(self) -> Dict: """ :return: A dict which value for the "policies" key is a dict which contains the "docs" part of the terms' YAML. That nested dict is empty if no terms. @@ -58,7 +59,7 @@ def getForClient(self): policies[docName].update(doc["langs"]) return {"policies": policies} - def getUrlSet(self): + def getUrlSet(self) -> Set[str]: """ :return: All the URLs for the terms in a set. Empty set if no terms. :rtype: set[unicode] @@ -76,7 +77,7 @@ def getUrlSet(self): urls.add(url) return urls - def urlListIsSufficient(self, urls): + def urlListIsSufficient(self, urls: List[str]) -> bool: """ Checks whether the provided list of URLs (which represents the list of terms accepted by the user) is enough to allow the creation of the user's account. @@ -102,7 +103,7 @@ def urlListIsSufficient(self, urls): return agreed == required -def get_terms(sydent): +def get_terms(sydent) -> Optional[Terms]: """Read and parse terms as specified in the config. :returns Terms @@ -139,3 +140,4 @@ def get_terms(sydent): logger.exception( "Couldn't read terms file '%s'", sydent.cfg.get("general", "terms.path") ) + return None # added per this: https://github.com/python/mypy/issues/3974 \ No newline at end of file diff --git a/sydent/threepid/bind.py b/sydent/threepid/bind.py index 641b1c3b..d01f3f86 100644 --- a/sydent/threepid/bind.py +++ b/sydent/threepid/bind.py @@ -40,7 +40,7 @@ class ThreepidBinder: # the lifetime of a 3pid association THREEPID_ASSOCIATION_LIFETIME_MS = 100 * 365 * 24 * 60 * 60 * 1000 - def __init__(self, sydent): + def __init__(self, sydent: sydent.Sydent) -> None: self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) From 6c215c85ae0ab3521ddebefba9fa4acc6340f1a3 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Thu, 3 Jun 2021 11:50:46 -0700 Subject: [PATCH 04/14] add type hints for mypy check --- sydent/db/hashing_metadata.py | 4 ++-- sydent/http/auth.py | 7 ++++--- sydent/threepid/bind.py | 19 ++++++++++++++----- sydent/threepid/signer.py | 11 ++++++++--- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index 25a3866e..dacebc47 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -17,7 +17,7 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py -from typing import Union, Callable, TYPE_CHECKING +from typing import Union, Callable, TYPE_CHECKING, Optional if TYPE_CHECKING: import sqlite3 @@ -27,7 +27,7 @@ class HashingMetadataStore: def __init__(self, sydent): self.sydent = sydent - def get_lookup_pepper(self) -> Union[str, None]: + def get_lookup_pepper(self) -> Optional[str]: """Return the value of the current lookup pepper from the db :return: A pepper if it exists in the database, or None if one does diff --git a/sydent/http/auth.py b/sydent/http/auth.py index 3b15db88..df502212 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -20,8 +20,9 @@ from sydent.db.accounts import AccountStore from sydent.http.servlets import MatrixRestError, get_args from sydent.terms.terms import get_terms +from sydent.sydent import Sydent -from typing import Union, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING if TYPE_CHECKING: from sydent.db.accounts import Account @@ -60,10 +61,10 @@ def tokenFromRequest(request: twisted.web.server.Request) -> Optional[str]: def authV2( - sydent: sydent.sydent.Sydent, + sydent: 'Sydent', request: twisted.web.server.Request, requireTermsAgreed: bool = True, -) -> Union[Account, None]: +) -> Optional[Account]: """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. diff --git a/sydent/threepid/bind.py b/sydent/threepid/bind.py index d01f3f86..8823d229 100644 --- a/sydent/threepid/bind.py +++ b/sydent/threepid/bind.py @@ -19,6 +19,8 @@ import collections import logging import math +import signedjson.sign # type: ignore +from sydent.db.invite_tokens import JoinTokenStore import signedjson.sign from twisted.internet import defer @@ -33,6 +35,13 @@ from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.stringutils import is_valid_matrix_server_name +from twisted.internet import defer + +from typing import TYPE_CHECKING, Dict, Any, Generator, Union, Optional + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) @@ -40,11 +49,11 @@ class ThreepidBinder: # the lifetime of a 3pid association THREEPID_ASSOCIATION_LIFETIME_MS = 100 * 365 * 24 * 60 * 60 * 1000 - def __init__(self, sydent: sydent.Sydent) -> None: + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) - def addBinding(self, medium, address, mxid): + def addBinding(self, medium: str, address: str, mxid: str) -> Dict[str, Any]: """ Binds the given 3pid to the given mxid. @@ -112,7 +121,7 @@ def addBinding(self, medium, address, mxid): return sgassoc - def removeBinding(self, threepid, mxid): + def removeBinding(self, threepid: Dict, mxid: str) -> None: """ Removes the binding between a given 3PID and a given MXID. @@ -126,7 +135,7 @@ def removeBinding(self, threepid, mxid): self.sydent.pusher.doLocalPush() @defer.inlineCallbacks - def _notify(self, assoc, attempt): + def _notify(self, assoc: Dict[str, Any], attempt: int) -> Generator: """ Sends data about a new association (and, if necessary, the associated invites) to the associated MXID's homeserver. @@ -193,7 +202,7 @@ def _notify(self, assoc, attempt): assoc["address"], ) - def _notifyErrback(self, assoc, attempt, error): + def _notifyErrback(self, assoc: Dict[str, Any], attempt: int, error: Union[Exception, str]) -> None: """ Handles errors when trying to send an association down to a homeserver by logging the error and scheduling a new attempt. diff --git a/sydent/threepid/signer.py b/sydent/threepid/signer.py index 848217c9..cfde63c0 100644 --- a/sydent/threepid/signer.py +++ b/sydent/threepid/signer.py @@ -14,14 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import signedjson.sign +import signedjson.sign # type: ignore +from typing import TYPE_CHECKING, Dict, Any + +if TYPE_CHECKING: + from sydent.sydent import Sydent + from sydent.threepid import ThreepidAssociation class Signer: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent - def signedThreePidAssociation(self, assoc): + def signedThreePidAssociation(self, assoc: 'ThreepidAssociation') -> Dict[str, Any]: """ Signs a 3PID association. From 13900867df8b793c97f798a83b3063e1cd1c2fb8 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 4 Jun 2021 10:35:31 -0700 Subject: [PATCH 05/14] add type hints to support mypy checks --- sydent/db/accounts.py | 12 +++++---- sydent/db/hashing_metadata.py | 7 ++--- sydent/db/invite_tokens.py | 9 ++++--- sydent/db/peers.py | 9 ++++--- sydent/db/sqlitedb.py | 7 ++++- sydent/db/terms.py | 7 +++-- sydent/db/threepid_associations.py | 26 +++++++++--------- sydent/db/valsession.py | 11 +++++--- sydent/hs_federation/verifier.py | 25 +++++++++-------- sydent/http/auth.py | 12 ++++----- sydent/http/httpclient.py | 17 ++++++------ sydent/http/httpcommon.py | 9 ++++++- sydent/http/httpsclient.py | 13 ++++++--- sydent/http/httpserver.py | 11 +++++--- sydent/http/matrixfederationagent.py | 30 ++++++++++++++------- sydent/http/srvresolver.py | 10 ++++--- sydent/replication/peer.py | 40 +++++++++++++++++++--------- sydent/replication/pusher.py | 17 +++++++----- sydent/sms/openmarket.py | 14 +++++++--- sydent/terms/terms.py | 10 +++---- sydent/threepid/bind.py | 12 +++++---- sydent/threepid/signer.py | 7 ++--- sydent/users/accounts.py | 2 +- sydent/users/tokens.py | 8 +++++- sydent/util/emailutils.py | 10 ++++++- sydent/util/hash.py | 4 +-- sydent/util/ip_range.py | 2 +- sydent/util/stringutils.py | 2 +- sydent/util/tokenutils.py | 6 ++--- sydent/util/ttlcache.py | 8 +++--- sydent/validators/common.py | 9 ++++++- sydent/validators/emailvalidator.py | 32 +++++++++++++++------- sydent/validators/msisdnvalidator.py | 27 ++++++++++++++----- 33 files changed, 276 insertions(+), 149 deletions(-) diff --git a/sydent/db/accounts.py b/sydent/db/accounts.py index dbed90e4..e5bd3bd2 100644 --- a/sydent/db/accounts.py +++ b/sydent/db/accounts.py @@ -17,14 +17,16 @@ from sydent.users.accounts import Account -from typing import Union +from typing import Optional, TYPE_CHECKING +if TYPE_CHECKING: + from sydent.sydent import Sydent class AccountStore(object): - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent - def getAccountByToken(self, token: str) -> Union[Account, None]: + def getAccountByToken(self, token: str) -> Optional[Account]: """ Select the account matching the given token, if any. @@ -48,7 +50,7 @@ def getAccountByToken(self, token: str) -> Union[Account, None]: return Account(*row) def storeAccount( - self, user_id: str, creation_ts: int, consent_version: Union[bytes, None] + self, user_id: str, creation_ts: int, consent_version: Optional[bytes] ) -> None: """ Stores an account for the given user ID. @@ -70,7 +72,7 @@ def storeAccount( self.sydent.db.commit() def setConsentVersion( - self, user_id: bytes, consent_version: Union[str, None] + self, user_id: bytes, consent_version: Optional[str] ) -> None: """ Saves that the given user has agreed to all of the terms in the document of the diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index dacebc47..40b2024b 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -16,15 +16,16 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py +from sqlite3 import Cursor from typing import Union, Callable, TYPE_CHECKING, Optional if TYPE_CHECKING: - import sqlite3 + from sydent.sydent import Sydent class HashingMetadataStore: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def get_lookup_pepper(self) -> Optional[str]: @@ -83,7 +84,7 @@ def store_lookup_pepper( def _rehash_threepids( self, - cur: "sqlite3.Cursor", + cur: "Cursor", hashing_function: Callable[[str], str], pepper: str, table: str, diff --git a/sydent/db/invite_tokens.py b/sydent/db/invite_tokens.py index c968883c..fcc9755b 100644 --- a/sydent/db/invite_tokens.py +++ b/sydent/db/invite_tokens.py @@ -15,11 +15,14 @@ # limitations under the License. import time -from typing import Union, List, Dict +from typing import Optional, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent class JoinTokenStore(object): - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def storeToken( @@ -163,7 +166,7 @@ def validateEphemeralPublicKey(self, publicKey: str) -> bool: self.sydent.db.commit() return cur.rowcount > 0 - def getSenderForToken(self, token: str) -> Union[str, None]: + def getSenderForToken(self, token: str) -> Optional[str]: """ Retrieves the MXID of the user that sent the invite the provided token is for. diff --git a/sydent/db/peers.py b/sydent/db/peers.py index 4aade6e1..4ba9b176 100644 --- a/sydent/db/peers.py +++ b/sydent/db/peers.py @@ -17,14 +17,17 @@ from sydent.replication.peer import RemotePeer -from typing import Union, List, Dict +from typing import Optional, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent class PeerStore: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent - def getPeerByName(self, name: str) -> Union[RemotePeer, None]: + def getPeerByName(self, name: str) -> Optional[RemotePeer]: """ Retrieves a remote peer using it's server name. diff --git a/sydent/db/sqlitedb.py b/sydent/db/sqlitedb.py index 4037cbc3..46e820ac 100644 --- a/sydent/db/sqlitedb.py +++ b/sydent/db/sqlitedb.py @@ -19,11 +19,16 @@ import os import sqlite3 +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) class SqliteDatabase: - def __init__(self, syd): + def __init__(self, syd: 'Sydent') -> None: self.sydent = syd dbFilePath = self.sydent.cfg.get("db", "db.file") diff --git a/sydent/db/terms.py b/sydent/db/terms.py index 974f74dc..d7682697 100644 --- a/sydent/db/terms.py +++ b/sydent/db/terms.py @@ -14,11 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List +from typing import List, TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent class TermsStore(object): - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def getAgreedUrls(self, user_id: str) -> List[str]: diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index 1ef41df0..b9e39766 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -19,21 +19,19 @@ from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer -<<<<<<< HEAD -from sydent.util import time_msec -======= import logging -from typing import Tuple, Union, List, Dict, Any, Sequence +from typing import Tuple, Optional, List, Dict, Any, Sequence, TYPE_CHECKING ->>>>>>> reset author +if TYPE_CHECKING: + from sydent.sydent import Sydent logger = logging.getLogger(__name__) class LocalAssociationStore: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: @@ -63,8 +61,8 @@ def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: self.sydent.db.commit() def getAssociationsAfterId( - self, afterId: int, limit: Union[int, None] = None - ) -> Tuple[Dict[int, ThreepidAssociation], Union[int, None]]: + self, afterId: int, limit: Optional[int] = None + ) -> Tuple[Dict[int, ThreepidAssociation], Optional[int]]: """ Retrieves every association after the given ID. @@ -108,8 +106,8 @@ def getAssociationsAfterId( return assocs, maxId def getSignedAssociationsAfterId( - self, afterId: int, limit: Union[int, None] = None - ) -> Tuple[Dict[int, Dict[str, Any]], Union[int, None]]: + self, afterId: int, limit: Optional[int] = None + ) -> Tuple[Dict[int, Dict[str, Any]], Optional[int]]: """Get associations after a given ID, and sign them before returning :param afterId: The ID to return results after (not inclusive) @@ -188,12 +186,12 @@ def removeAssociation(self, threepid: Dict[str, str], mxid: str) -> None: class GlobalAssociationStore: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def signedAssociationStringForThreepid( self, medium: str, address: str - ) -> Union[str, None]: + ) -> Optional[str]: """ Retrieve the JSON for the signed association matching the provided 3PID, if one exists. @@ -228,7 +226,7 @@ def signedAssociationStringForThreepid( return sgAssocStr - def getMxid(self, medium: str, address: str) -> Union[str, None]: + def getMxid(self, medium: str, address: str) -> Optional[str]: """ Retrieves the MXID associated with a 3PID. @@ -356,7 +354,7 @@ def addAssociation( if commit: self.sydent.db.commit() - def lastIdFromServer(self, server: str) -> Union[int, None]: + def lastIdFromServer(self, server: str) -> Optional[int]: """ Retrieves the ID of the last association received from the given peer. diff --git a/sydent/db/valsession.py b/sydent/db/valsession.py index e944c432..b8afb227 100644 --- a/sydent/db/valsession.py +++ b/sydent/db/valsession.py @@ -27,11 +27,14 @@ ValidationSession, ) -from typing import Union +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from sydent.sydent import Sydent class ThreePidValSessionStore: - def __init__(self, syd): + def __init__(self, syd: 'Sydent') -> None: self.sydent = syd self.random = SystemRandom() @@ -179,7 +182,7 @@ def setMtime(self, sid: str, mtime: int) -> None: ) self.sydent.db.commit() - def getSessionById(self, sid: str) -> Union[ValidationSession, None]: + def getSessionById(self, sid: str) -> Optional[ValidationSession]: """ Retrieves the session matching the given sid. @@ -206,7 +209,7 @@ def getSessionById(self, sid: str) -> Union[ValidationSession, None]: row[0], row[1], row[2], row[3], row[4], row[5], None, None ) - def getTokenSessionById(self, sid: str) -> Union[ValidationSession, None]: + def getTokenSessionById(self, sid: str) -> Optional[ValidationSession]: """ Retrieves a validation session using the session's ID. diff --git a/sydent/hs_federation/verifier.py b/sydent/hs_federation/verifier.py index ca80164d..146f84a8 100644 --- a/sydent/hs_federation/verifier.py +++ b/sydent/hs_federation/verifier.py @@ -22,9 +22,10 @@ import signedjson.key import signedjson.sign from twisted.internet import defer -from unpaddedbase64 import decode_base64 # type: ignore -import signedjson.sign #type: ignore -import signedjson.key #type: ignore +from twisted.web.server import Request +from unpaddedbase64 import decode_base64 # type: ignore +import signedjson.sign # type: ignore +import signedjson.key # type: ignore from signedjson.sign import SignatureVerifyException from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -32,10 +33,10 @@ from sydent.http.httpclient import FederationHttpClient from sydent.util.stringutils import is_valid_matrix_server_name -from typing import Dict, Union, TYPE_CHECKING, Any, List, Tuple, Generator +from typing import Dict, Optional, TYPE_CHECKING, Any, List, Tuple, Generator if TYPE_CHECKING: - import twisted + from sydent.sydent import Sydent logger = logging.getLogger(__name__) @@ -63,18 +64,16 @@ class Verifier(object): verifying that the signature on the json blob matches. """ - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent # Cache of server keys. These are cached until the 'valid_until_ts' time # in the result. - self.cache = { + self.cache: Dict = { # server_name: , } @defer.inlineCallbacks - def _getKeysForServer( - self, server_name: str - ) -> Generator: + def _getKeysForServer(self, server_name: str) -> Generator: """Get the signing key data from a homeserver. :param server_name: The name of the server to request the keys from. @@ -120,7 +119,7 @@ def _getKeysForServer( def verifyServerSignedJson( self, signed_json: Dict[str, Any], - acceptable_server_names: Union[List[str], None] = None, + acceptable_server_names: Optional[List[str]] = None, ) -> Generator: """Given a signed json object, try to verify any one of the signatures on it @@ -179,7 +178,7 @@ def verifyServerSignedJson( @defer.inlineCallbacks def authenticate_request( - self, request: twisted.web.server.Request, content: Union[bytes, None] + self, request: 'Request', content: Optional[bytes] ) -> Generator: """Authenticates a Matrix federation request based on the X-Matrix header XXX: Copied largely from synapse @@ -217,7 +216,7 @@ def parse_auth_header(header_str: str) -> Tuple[str, str, str]: """ try: params = header_str.split(u" ")[1].split(u",") - param_dict: Dict[str, str] = dict(kv.split(u"=") for kv in params) + param_dict = dict(kv.split(u"=") for kv in params) def strip_quotes(value): if value.startswith(u'"'): diff --git a/sydent/http/auth.py b/sydent/http/auth.py index df502212..fdb05555 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -22,18 +22,18 @@ from sydent.terms.terms import get_terms from sydent.sydent import Sydent +from twisted.web.server import Request + from typing import Optional, TYPE_CHECKING if TYPE_CHECKING: from sydent.db.accounts import Account - import twisted from sydent.sydent import Sydent - logger = logging.getLogger(__name__) -def tokenFromRequest(request: twisted.web.server.Request) -> Optional[str]: +def tokenFromRequest(request: 'Request') -> Optional[str]: """Extract token from header of query parameter. :param request: The request to look for an access token in. @@ -61,10 +61,10 @@ def tokenFromRequest(request: twisted.web.server.Request) -> Optional[str]: def authV2( - sydent: 'Sydent', - request: twisted.web.server.Request, + sydent: "Sydent", + request: 'Request', requireTermsAgreed: bool = True, -) -> Optional[Account]: +) -> Optional['Account']: """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py index 5445f395..a339ff28 100644 --- a/sydent/http/httpclient.py +++ b/sydent/http/httpclient.py @@ -22,6 +22,7 @@ from twisted.internet import defer from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers +from twisted.web.iweb import IAgent from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper from sydent.http.federation_tls_options import ClientTLSOptionsFactory @@ -29,10 +30,10 @@ from sydent.http.matrixfederationagent import MatrixFederationAgent from sydent.util import json_decoder -from typing import Union, Dict, Any, TYPE_CHECKING, Generator +from typing import Optional, Dict, Any, TYPE_CHECKING, Generator if TYPE_CHECKING: - import twisted + from sydent.sydent import Sydent logger = logging.getLogger(__name__) @@ -41,13 +42,13 @@ class HTTPClient(object): """A base HTTP class that contains methods for making GET and POST HTTP requests. """ - agent: Any # TODO: find actual type of twisted.web.iweb.IAgent + agent: Any # TODO: find type that will work with both Agent() and MatrixFederationAgent() @defer.inlineCallbacks def get_json( - self, uri: str, max_size: Union[int, None] = None - ) -> Generator[Dict[Any, Any], Any, Any]: + self, uri: str, max_size: Optional[int] = None + ) -> Generator: """Make a GET request to an endpoint returning JSON and parse result :param uri: The URI to make a GET request to. @@ -77,7 +78,7 @@ def get_json( @defer.inlineCallbacks def post_json_get_nothing( self, uri: str, post_json: Dict[Any, Any], opts: Dict[str, Any] - ) -> Generator[Any, Any, Any]: + ) -> Generator: """Make a POST request to an endpoint returning JSON and parse result :param uri: The URI to make a POST request to. @@ -131,7 +132,7 @@ class SimpleHttpClient(HTTPClient): from Synapse. """ - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent # The default endpoint factory in Twisted 14.0.0 (which we require) uses the # BrowserLikePolicyForHTTPS context factory which will do regular cert validation @@ -151,7 +152,7 @@ class FederationHttpClient(HTTPClient): MatrixFederationAgent. """ - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.agent = MatrixFederationAgent( BlacklistingReactorWrapper( diff --git a/sydent/http/httpcommon.py b/sydent/http/httpcommon.py index 28fbd844..cea224ff 100644 --- a/sydent/http/httpcommon.py +++ b/sydent/http/httpcommon.py @@ -24,6 +24,13 @@ from twisted.web._newclient import ResponseDone from twisted.web.http import PotentialDataLoss from twisted.web.iweb import UNKNOWN_LENGTH +from twisted.web import server + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) @@ -32,7 +39,7 @@ class SslComponents: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.myPrivateCertificate = self.makeMyCertificate() diff --git a/sydent/http/httpsclient.py b/sydent/http/httpsclient.py index 442b208c..f1f894b2 100644 --- a/sydent/http/httpsclient.py +++ b/sydent/http/httpsclient.py @@ -25,6 +25,11 @@ from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer +from typing import TYPE_CHECKING, Dict, Any, Generator, Optional + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) @@ -35,7 +40,7 @@ class ReplicationHttpsClient: replication HTTPS server) """ - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.agent = None @@ -47,7 +52,7 @@ def __init__(self, sydent): # trustRoot=self.sydent.sslComponents.trustRoot) self.agent = Agent(self.sydent.reactor, SydentPolicyForHTTPS(self.sydent)) - def postJson(self, uri, jsonObject): + def postJson(self, uri: str, jsonObject: Dict[Any, Any]) -> Optional[Generator]: """ Sends an POST request over HTTPS. @@ -62,7 +67,7 @@ def postJson(self, uri, jsonObject): logger.debug("POSTing request to %s", uri) if not self.agent: logger.error("HTTPS post attempted but HTTPS is not configured") - return + return None headers = Headers( {"Content-Type": ["application/json"], "User-Agent": ["Sydent"]} @@ -78,7 +83,7 @@ def postJson(self, uri, jsonObject): @implementer(IPolicyForHTTPS) class SydentPolicyForHTTPS(object): - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def creatorForNetloc(self, hostname, port): diff --git a/sydent/http/httpserver.py b/sydent/http/httpserver.py index 6ee7d72b..640254a4 100644 --- a/sydent/http/httpserver.py +++ b/sydent/http/httpserver.py @@ -31,11 +31,16 @@ AuthenticatedUnbindThreePidServlet, ) +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) class ClientApiHttpServer: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent root = Resource() @@ -149,7 +154,7 @@ def setup(self): class InternalApiHttpServer(object): - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent def setup(self, interface, port): @@ -177,7 +182,7 @@ def setup(self, interface, port): class ReplicationHttpsServer: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent root = Resource() diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py index 5662fb67..9d32dcfd 100644 --- a/sydent/http/matrixfederationagent.py +++ b/sydent/http/matrixfederationagent.py @@ -19,21 +19,26 @@ import time import attr -from netaddr import IPAddress +from netaddr import IPAddress # type: ignore +from zope.interface import implementer + from twisted.internet import defer +from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet.interfaces import IStreamClientEndpoint from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent from twisted.web.http import stringToDatetime from twisted.web.http_headers import Headers -from twisted.web.iweb import IAgent -from zope.interface import implementer +from twisted.web.iweb import IAgent, IBodyProducer + from sydent.http.httpcommon import read_body_with_max_size from sydent.http.srvresolver import SrvResolver, pick_server_from_list from sydent.util import json_decoder from sydent.util.ttlcache import TTLCache +from typing import Generator, Optional + # period to cache .well-known results for by default WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600 @@ -84,9 +89,9 @@ def __init__( reactor, tls_client_options_factory, _well_known_tls_policy=None, - _srv_resolver=None, - _well_known_cache=well_known_cache, - ): + _srv_resolver: Optional['SrvResolver']=None, + _well_known_cache: Optional['TTLCache']=well_known_cache, + ) -> None: self._reactor = reactor self._tls_client_options_factory = tls_client_options_factory @@ -116,7 +121,12 @@ def __init__( self._well_known_cache = _well_known_cache @defer.inlineCallbacks - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional['Headers']=None, + bodyProducer: Optional['IBodyProducer']=None) -> 'Deferred': """ :param method: HTTP method (GET/POST/etc). :type method: bytes @@ -181,7 +191,7 @@ def endpointForURI(_uri): defer.returnValue(res) @defer.inlineCallbacks - def _route_matrix_uri(self, parsed_uri, lookup_well_known=True): + def _route_matrix_uri(self, parsed_uri: 'URI', lookup_well_known: bool=True) -> 'Deferred': """Helper for `request`: determine the routing for a Matrix URI :param parsed_uri: uri to route. Note that it should be parsed with @@ -294,7 +304,7 @@ def _route_matrix_uri(self, parsed_uri, lookup_well_known=True): ) @defer.inlineCallbacks - def _get_well_known(self, server_name): + def _get_well_known(self, server_name: bytes) -> 'Deferred': """Attempt to fetch and parse a .well-known file for the given server :param server_name: Name of the server, from the requested url. @@ -317,7 +327,7 @@ def _get_well_known(self, server_name): defer.returnValue(result) @defer.inlineCallbacks - def _do_get_well_known(self, server_name): + def _do_get_well_known(self, server_name: bytes) -> 'Deferred': """Actually fetch and parse a .well-known, without checking the cache :param server_name: Name of the server, from the requested url diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index 586bd85f..08f42d07 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -20,10 +20,14 @@ import attr from twisted.internet import defer +from twisted.internet.defer import Deferred from twisted.internet.error import ConnectError +from twisted.internet.interfaces import IResolver from twisted.names import client, dns from twisted.names.error import DNSNameError, DomainError +from typing import Callable, Dict, Tuple + logger = logging.getLogger(__name__) SERVER_CACHE = {} @@ -49,7 +53,7 @@ class Server(object): expires = attr.ib(default=0) -def pick_server_from_list(server_list): +def pick_server_from_list(server_list: list[Server]) -> Tuple[bytes, int]: """Randomly choose a server from the server list. :param server_list: List of candidate servers. @@ -96,13 +100,13 @@ class SrvResolver(object): :type get_time: callable """ - def __init__(self, dns_client=client, cache=SERVER_CACHE, get_time=time.time): + def __init__(self, dns_client: 'IResolver'=client, cache: Dict=SERVER_CACHE, get_time: Callable=time.time) -> None: self._dns_client = dns_client self._cache = cache self._get_time = get_time @defer.inlineCallbacks - def resolve_service(self, service_name): + def resolve_service(self, service_name: bytes) -> 'Deferred': """Look up a SRV record :param service_name: The record to look up. diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py index b06b668b..ff103555 100644 --- a/sydent/replication/peer.py +++ b/sydent/replication/peer.py @@ -33,20 +33,24 @@ from sydent.threepid import threePidAssocFromDict from sydent.util import json_decoder from sydent.util.hash import sha256_and_url_safe_base64 -from unpaddedbase64 import decode_base64 # type: ignore +from unpaddedbase64 import decode_base64 # type: ignore -import signedjson.sign # type: ignore -import signedjson.key # type: ignore +import signedjson.sign # type: ignore +import signedjson.key # type: ignore import logging import json import binascii from twisted.internet import defer +from twisted.internet.defer import Deferred from twisted.web.client import readBody +from twisted.web.iweb import IResponse -from collections.abc import Sequence -from typing import Generator +from typing import Generator, Dict, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent logger = logging.getLogger(__name__) @@ -59,7 +63,7 @@ def __init__(self, servername, pubkeys): self.pubkeys = pubkeys self.is_being_pushed_to = False - def pushUpdates(self, sgAssocs: Sequence[tuple]) -> Generator: + def pushUpdates(self, sgAssocs) -> 'Deferred': """ :param sgAssocs: Sequence of (originId, sgAssoc) tuples where originId is the id on the creating server and sgAssoc is the json object of the signed association @@ -73,7 +77,7 @@ class LocalPeer(Peer): The local peer (ourselves: essentially copying from the local associations table to the global one) """ - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: super(LocalPeer, self).__init__(sydent.server_name, {}) self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) @@ -83,7 +87,7 @@ def __init__(self, sydent): if self.lastId is None: self.lastId = -1 - def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': """ Saves the given associations in the global associations store. Only stores an association if its ID is greater than the last seen ID. @@ -128,7 +132,7 @@ def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: class RemotePeer(Peer): - def __init__(self, sydent, server_name, port, pubkeys, lastSentVersion): + def __init__(self, sydent: 'Sydent', server_name: str, port: int, pubkeys: Dict[str, str], lastSentVersion: int) -> None: """ :param sydent: The current Sydent instance. :type sydent: sydent.sydent.Sydent @@ -219,7 +223,7 @@ def verifySignedAssociation(self, assoc: Dict[Any, Any]) -> None: # Verify the JSON signedjson.sign.verify_signed_json(assoc, self.servername, self.verify_key) - def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': """ Pushes the given associations to the peer. @@ -247,7 +251,11 @@ def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> Generator: return updateDeferred - def _pushSuccess(self, result: twisted.web.iweb.IResponse, updateDeferred: twisted.internet.defer.Deferred) -> None: + def _pushSuccess( + self, + result: 'IResponse', + updateDeferred: 'Deferred', + ) -> None: """ Processes a successful push request. If the request resulted in a status code that's not a success, consider it a failure @@ -265,7 +273,9 @@ def _pushSuccess(self, result: twisted.web.iweb.IResponse, updateDeferred: twist d.addCallback(self._failedPushBodyRead, updateDeferred=updateDeferred) d.addErrback(self._pushFailed, updateDeferred=updateDeferred) - def _failedPushBodyRead(self, body: bytes, updateDeferred: twisted.internet.defer.Deferred) -> None: + def _failedPushBodyRead( + self, body: bytes, updateDeferred: 'Deferred' + ) -> None: """ Processes a response body from a failed push request, then calls the error callback of the provided deferred. @@ -280,7 +290,11 @@ def _failedPushBodyRead(self, body: bytes, updateDeferred: twisted.internet.defe e.errorDict = errObj updateDeferred.errback(e) - def _pushFailed(self, failure: twisted.python.failure.Failure, updateDeferred: twisted.internet.defer.Deferred) -> None: + def _pushFailed( + self, + failure, + updateDeferred: 'Deferred', + ) -> None: """ Processes a failed push request, by calling the error callback of the given deferred with it. diff --git a/sydent/replication/pusher.py b/sydent/replication/pusher.py index adec54b8..1b12df5e 100644 --- a/sydent/replication/pusher.py +++ b/sydent/replication/pusher.py @@ -22,10 +22,15 @@ import twisted.internet.task from twisted.internet import defer -from sydent.db.peers import PeerStore -from sydent.db.threepid_associations import LocalAssociationStore -from sydent.replication.peer import LocalPeer from sydent.util import time_msec +from sydent.replication.peer import LocalPeer, RemotePeer +from sydent.db.threepid_associations import LocalAssociationStore +from sydent.db.peers import PeerStore + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent logger = logging.getLogger(__name__) @@ -34,7 +39,7 @@ class Pusher: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.pushing = False self.peerStore = PeerStore(self.sydent) @@ -45,7 +50,7 @@ def setup(self): cb.clock = self.sydent.reactor cb.start(10.0) - def doLocalPush(self): + def doLocalPush(self) -> None: """ Synchronously push local associations to this server (ie. copy them to globals table) The local server is essentially treated the same as any other peer except we don't do @@ -74,7 +79,7 @@ def scheduledPush(self): return defer.DeferredList([self._push_to_peer(p) for p in peers]) @defer.inlineCallbacks - def _push_to_peer(self, p): + def _push_to_peer(self, p: 'RemotePeer') -> None: """ For a given peer, retrieves the list of associations that were created since the last successful push to this peer (limited to ASSOCIATIONS_PUSH_LIMIT) and diff --git a/sydent/sms/openmarket.py b/sydent/sms/openmarket.py index 3f35c413..bcd46ae9 100644 --- a/sydent/sms/openmarket.py +++ b/sydent/sms/openmarket.py @@ -21,9 +21,13 @@ from twisted.internet import defer from twisted.web.http_headers import Headers + from sydent.http.httpclient import SimpleHttpClient -from typing import Union, Dict, Generator, Any +from typing import Dict, Any, TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from sydent.sydent import Sydent logger = logging.getLogger(__name__) @@ -42,7 +46,7 @@ } -def tonFromType(t): +def tonFromType(t: str) -> int: """ Get the type of number from the originator's type. @@ -58,12 +62,14 @@ def tonFromType(t): class OpenMarketSMS: - def __init__(self, sydent): + def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent self.http_cli = SimpleHttpClient(sydent) @defer.inlineCallbacks - def sendTextSMS(self, body: Dict, dest: str, source: Union[None, Dict[str, str]]=None) -> Generator: + def sendTextSMS( + self, body: Dict, dest: str, source: Optional[Dict[str, str]] = None + ) -> None: """ Sends a text message with the given body to the given MSISDN. diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index 370f729c..4a2b2774 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -17,20 +17,20 @@ import logging import yaml -from typing import Union, Set, List, Dict, Optional +from typing import Any, Set, List, Dict, Optional logger = logging.getLogger(__name__) class Terms(object): - def __init__(self, yamlObj) -> None: + def __init__(self, yamlObj: Optional[Dict[str, Any]]) -> None: """ :param yamlObj: The parsed YAML. :type yamlObj: dict[str, any] or None """ self._rawTerms = yamlObj - def getMasterVersion(self) -> Union[str, None]: + def getMasterVersion(self) -> Optional[str]: """ :return: The global (master) version of the terms, or None if there are no terms of service for this server. @@ -44,7 +44,7 @@ def getMasterVersion(self) -> Union[str, None]: return version - def getForClient(self) -> Dict: + def getForClient(self) -> Dict[str, dict]: """ :return: A dict which value for the "policies" key is a dict which contains the "docs" part of the terms' YAML. That nested dict is empty if no terms. @@ -140,4 +140,4 @@ def get_terms(sydent) -> Optional[Terms]: logger.exception( "Couldn't read terms file '%s'", sydent.cfg.get("general", "terms.path") ) - return None # added per this: https://github.com/python/mypy/issues/3974 \ No newline at end of file + return None # added per this: https://github.com/python/mypy/issues/3974 diff --git a/sydent/threepid/bind.py b/sydent/threepid/bind.py index 8823d229..da68f740 100644 --- a/sydent/threepid/bind.py +++ b/sydent/threepid/bind.py @@ -19,7 +19,7 @@ import collections import logging import math -import signedjson.sign # type: ignore +import signedjson.sign # type: ignore from sydent.db.invite_tokens import JoinTokenStore import signedjson.sign @@ -49,7 +49,7 @@ class ThreepidBinder: # the lifetime of a 3pid association THREEPID_ASSOCIATION_LIFETIME_MS = 100 * 365 * 24 * 60 * 60 * 1000 - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) @@ -121,7 +121,7 @@ def addBinding(self, medium: str, address: str, mxid: str) -> Dict[str, Any]: return sgassoc - def removeBinding(self, threepid: Dict, mxid: str) -> None: + def removeBinding(self, threepid: Dict[str, str], mxid: str) -> None: """ Removes the binding between a given 3PID and a given MXID. @@ -135,7 +135,7 @@ def removeBinding(self, threepid: Dict, mxid: str) -> None: self.sydent.pusher.doLocalPush() @defer.inlineCallbacks - def _notify(self, assoc: Dict[str, Any], attempt: int) -> Generator: + def _notify(self, assoc: Dict[str, Any], attempt: int) -> None: """ Sends data about a new association (and, if necessary, the associated invites) to the associated MXID's homeserver. @@ -202,7 +202,9 @@ def _notify(self, assoc: Dict[str, Any], attempt: int) -> Generator: assoc["address"], ) - def _notifyErrback(self, assoc: Dict[str, Any], attempt: int, error: Union[Exception, str]) -> None: + def _notifyErrback( + self, assoc: Dict[str, Any], attempt: int, error: Union[Exception, str] + ) -> None: """ Handles errors when trying to send an association down to a homeserver by logging the error and scheduling a new attempt. diff --git a/sydent/threepid/signer.py b/sydent/threepid/signer.py index cfde63c0..78101cc5 100644 --- a/sydent/threepid/signer.py +++ b/sydent/threepid/signer.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import signedjson.sign # type: ignore +import signedjson.sign # type: ignore from typing import TYPE_CHECKING, Dict, Any @@ -22,11 +22,12 @@ from sydent.sydent import Sydent from sydent.threepid import ThreepidAssociation + class Signer: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent - def signedThreePidAssociation(self, assoc: 'ThreepidAssociation') -> Dict[str, Any]: + def signedThreePidAssociation(self, assoc: "ThreepidAssociation") -> Dict[str, Any]: """ Signs a 3PID association. diff --git a/sydent/users/accounts.py b/sydent/users/accounts.py index 4bef44b0..c022230a 100644 --- a/sydent/users/accounts.py +++ b/sydent/users/accounts.py @@ -15,7 +15,7 @@ class Account(object): - def __init__(self, user_id, creation_ts, consent_version): + def __init__(self, user_id: str, creation_ts: int, consent_version: str) -> None: """ :param user_id: The Matrix user ID for the account. :type user_id: str diff --git a/sydent/users/tokens.py b/sydent/users/tokens.py index 30c837a4..0bc99dca 100644 --- a/sydent/users/tokens.py +++ b/sydent/users/tokens.py @@ -20,10 +20,16 @@ from sydent.db.accounts import AccountStore from sydent.util.tokenutils import generateAlphanumericTokenOfLength +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sydent.sydent import Sydent + + logger = logging.getLogger(__name__) -def issueToken(sydent, user_id): +def issueToken(sydent: "Sydent", user_id: str) -> str: """ Creates an account for the given Matrix user ID, then generates, saves and returns an access token for that account. diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 9d4c789e..72a8dbcc 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -34,10 +34,17 @@ from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength +from typing import TYPE_CHECKING, Dict, Any + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) -def sendEmail(sydent, templateFile, mailTo, substitutions): +def sendEmail( + sydent: "Sydent", templateFile: str, mailTo: str, substitutions: Dict[str, str] +) -> None: """ Sends an email with the given parameters. @@ -131,4 +138,5 @@ class EmailAddressException(Exception): class EmailSendException(Exception): + cause: Any # type hint added to prevent ""EmailSendException" has no attribute "cause"" error in Mypy pass diff --git a/sydent/util/hash.py b/sydent/util/hash.py index 799094e8..73328dce 100644 --- a/sydent/util/hash.py +++ b/sydent/util/hash.py @@ -16,10 +16,10 @@ import hashlib -import unpaddedbase64 +import unpaddedbase64 # type: ignore -def sha256_and_url_safe_base64(input_text): +def sha256_and_url_safe_base64(input_text: str) -> bytes: """SHA256 hash an input string, encode the digest as url-safe base64, and return diff --git a/sydent/util/ip_range.py b/sydent/util/ip_range.py index 9e81a63e..f747b2c6 100644 --- a/sydent/util/ip_range.py +++ b/sydent/util/ip_range.py @@ -15,7 +15,7 @@ import itertools from typing import Iterable, Optional -from netaddr import AddrFormatError, IPNetwork, IPSet +from netaddr import AddrFormatError, IPNetwork, IPSet # type: ignore # IP ranges that are considered private / unroutable / don't make sense. DEFAULT_IP_RANGE_BLACKLIST = [ diff --git a/sydent/util/stringutils.py b/sydent/util/stringutils.py index 3b9f66fe..f526a698 100644 --- a/sydent/util/stringutils.py +++ b/sydent/util/stringutils.py @@ -37,7 +37,7 @@ MAX_EMAIL_ADDRESS_LENGTH = 500 -def is_valid_client_secret(client_secret): +def is_valid_client_secret(client_secret: str) -> bool: """Validate that a given string matches the client_secret regex defined by the spec :param client_secret: The client_secret to validate diff --git a/sydent/util/tokenutils.py b/sydent/util/tokenutils.py index 8800f708..4e8ec3f4 100644 --- a/sydent/util/tokenutils.py +++ b/sydent/util/tokenutils.py @@ -20,7 +20,7 @@ r = random.SystemRandom() -def generateTokenForMedium(medium): +def generateTokenForMedium(medium: str) -> str: """ Generates a token of a different format depending on the medium, a 32 characters alphanumeric one if the medium is email, a 6 characters numeric one otherwise. @@ -37,7 +37,7 @@ def generateTokenForMedium(medium): return generateNumericTokenOfLength(6) -def generateNumericTokenOfLength(length): +def generateNumericTokenOfLength(length: int) -> str: """ Generates a token of the given length with the character set [0-9]. @@ -50,7 +50,7 @@ def generateNumericTokenOfLength(length): return u"".join([r.choice(string.digits) for _ in range(length)]) -def generateAlphanumericTokenOfLength(length): +def generateAlphanumericTokenOfLength(length: int) -> str: """ Generates a token of the given length with the character set [a-zA-Z0-9]. diff --git a/sydent/util/ttlcache.py b/sydent/util/ttlcache.py index 8d98c301..b24232c8 100644 --- a/sydent/util/ttlcache.py +++ b/sydent/util/ttlcache.py @@ -17,7 +17,9 @@ import time import attr -from sortedcontainers import SortedList +from sortedcontainers import SortedList # type: ignore + +from typing import Any, Dict, Tuple logger = logging.getLogger(__name__) @@ -36,7 +38,7 @@ def __init__(self, cache_name, timer=time.time): self._timer = timer - def set(self, key, value, ttl): + def set(self, key, value, ttl: float) -> None: """Add/update an entry in the cache :param key: Key for this entry. @@ -74,7 +76,7 @@ def get(self, key, default=SENTINEL): return default return e.value - def get_with_expiry(self, key): + def get_with_expiry(self, key) -> Tuple[Any, float]: """Get a value, and its expiry time, from the cache :param key: key to look up diff --git a/sydent/validators/common.py b/sydent/validators/common.py index 787eccde..49bf29de 100644 --- a/sydent/validators/common.py +++ b/sydent/validators/common.py @@ -12,10 +12,17 @@ ValidationSession, ) +from typing import TYPE_CHECKING, Dict + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) -def validateSessionWithToken(sydent, sid, clientSecret, token): +def validateSessionWithToken( + sydent: "Sydent", sid: str, clientSecret: str, token: str +) -> Dict[str, bool]: """ Attempt to validate a session, identified by the sid, using the token from out-of-band. The client secret is given to diff --git a/sydent/validators/emailvalidator.py b/sydent/validators/emailvalidator.py index 959af527..6ee292bc 100644 --- a/sydent/validators/emailvalidator.py +++ b/sydent/validators/emailvalidator.py @@ -24,22 +24,30 @@ from sydent.util.emailutils import sendEmail from sydent.validators import common +from sydent.util import time_msec + +from typing import TYPE_CHECKING, Optional, Dict + +if TYPE_CHECKING: + from sydent.sydent import Sydent + from sydent.validators import ValidationSession + logger = logging.getLogger(__name__) class EmailValidator: - def __init__(self, sydent): + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def requestToken( self, - emailAddress, - clientSecret, - sendAttempt, - nextLink, - ipaddress=None, - brand=None, - ): + emailAddress: str, + clientSecret: str, + sendAttempt: int, + nextLink: str, + ipaddress: Optional[str] = None, + brand: Optional[str] = None, + ) -> int: """ Creates or retrieves a validation session and sends an email to the corresponding email address with a token to use to verify the association. @@ -102,7 +110,9 @@ def requestToken( return valSession.id - def makeValidateLink(self, valSession, clientSecret, nextLink): + def makeValidateLink( + self, valSession: "ValidationSession", clientSecret: str, nextLink: str + ) -> str: """ Creates a validation link that can be sent via email to the user. @@ -137,7 +147,9 @@ def makeValidateLink(self, valSession, clientSecret, nextLink): link += "&nextLink=%s" % (urllib.parse.quote(nextLink)) return link - def validateSessionWithToken(self, sid, clientSecret, token): + def validateSessionWithToken( + self, sid: str, clientSecret: str, token: str + ) -> Dict[str, bool]: """ Validates the session with the given ID. diff --git a/sydent/validators/msisdnvalidator.py b/sydent/validators/msisdnvalidator.py index ada2cbdb..3dbf1220 100644 --- a/sydent/validators/msisdnvalidator.py +++ b/sydent/validators/msisdnvalidator.py @@ -18,23 +18,28 @@ import logging -import phonenumbers +import phonenumbers # type: ignore from sydent.db.valsession import ThreePidValSessionStore from sydent.sms.openmarket import OpenMarketSMS from sydent.util import time_msec from sydent.validators import DestinationRejectedException, common +from typing import TYPE_CHECKING, Optional, Dict + +if TYPE_CHECKING: + from sydent.sydent import Sydent + logger = logging.getLogger(__name__) class MsisdnValidator: - def __init__(self, sydent): + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.omSms = OpenMarketSMS(sydent) # cache originators & sms rules from config file - self.originators = {} + self.originators: Dict = {} self.smsRules = {} for opt in self.sydent.cfg.options("sms"): if opt.startswith("originators."): @@ -71,7 +76,13 @@ def __init__(self, sydent): self.smsRules[country] = action - def requestToken(self, phoneNumber, clientSecret, sendAttempt, brand=None): + def requestToken( + self, + phoneNumber: phonenumbers.PhoneNumber, + clientSecret: str, + sendAttempt: int, + brand: Optional[str] = None, + ) -> int: """ Creates or retrieves a validation session and sends an text message to the corresponding phone number address with a token to use to verify the association. @@ -132,7 +143,9 @@ def requestToken(self, phoneNumber, clientSecret, sendAttempt, brand=None): return valSession.id - def getOriginator(self, destPhoneNumber): + def getOriginator( + self, destPhoneNumber: phonenumbers.PhoneNumber + ) -> Dict[str, str]: """ Gets an originator for a given phone number. @@ -165,7 +178,9 @@ def getOriginator(self, destPhoneNumber): )[1:] return origs[sum([int(i) for i in msisdn]) % len(origs)] - def validateSessionWithToken(self, sid, clientSecret, token): + def validateSessionWithToken( + self, sid: str, clientSecret: str, token: str + ) -> Dict[str, bool]: """ Validates the session with the given ID. From e0fdbe8da0efea3cd71b8cf6f3d3815504d44c83 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 4 Jun 2021 12:36:58 -0700 Subject: [PATCH 06/14] fix list import from typing --- sydent/http/srvresolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index 08f42d07..a6ca1dd5 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -26,7 +26,7 @@ from twisted.names import client, dns from twisted.names.error import DNSNameError, DomainError -from typing import Callable, Dict, Tuple +from typing import Callable, Dict, Tuple, List logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class Server(object): expires = attr.ib(default=0) -def pick_server_from_list(server_list: list[Server]) -> Tuple[bytes, int]: +def pick_server_from_list(server_list: List[Server]) -> Tuple[bytes, int]: """Randomly choose a server from the server list. :param server_list: List of candidate servers. From 3029212f9a7622530317a92d038988f57c9a9181 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 4 Jun 2021 14:13:20 -0700 Subject: [PATCH 07/14] fixed import and added changelog entry --- sydent/http/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sydent/http/auth.py b/sydent/http/auth.py index fdb05555..16825a23 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -20,7 +20,6 @@ from sydent.db.accounts import AccountStore from sydent.http.servlets import MatrixRestError, get_args from sydent.terms.terms import get_terms -from sydent.sydent import Sydent from twisted.web.server import Request From 93981289d53361644051ad0b77e526459ce99604 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Fri, 4 Jun 2021 14:17:28 -0700 Subject: [PATCH 08/14] add changelog --- changelog.d/355.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/355.misc diff --git a/changelog.d/355.misc b/changelog.d/355.misc new file mode 100644 index 00000000..b1c52a2b --- /dev/null +++ b/changelog.d/355.misc @@ -0,0 +1 @@ +Added type hints to support mypy checks. \ No newline at end of file From fae2511bcc25259930018bcbc4d84bc30081a758 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Mon, 7 Jun 2021 10:42:23 -0700 Subject: [PATCH 09/14] requested changes --- sydent/db/accounts.py | 4 ++-- sydent/db/hashing_metadata.py | 2 +- sydent/db/threepid_associations.py | 10 +++++----- sydent/db/valsession.py | 12 ++++++------ sydent/hs_federation/verifier.py | 2 +- sydent/http/auth.py | 2 +- sydent/http/httpclient.py | 2 +- sydent/http/httpsclient.py | 5 +++-- sydent/http/matrixfederationagent.py | 6 +++--- sydent/http/srvresolver.py | 6 +++--- sydent/replication/pusher.py | 4 ++-- sydent/terms/terms.py | 2 +- sydent/util/hash.py | 2 +- sydent/validators/msisdnvalidator.py | 2 +- 14 files changed, 31 insertions(+), 30 deletions(-) diff --git a/sydent/db/accounts.py b/sydent/db/accounts.py index e5bd3bd2..68f0a522 100644 --- a/sydent/db/accounts.py +++ b/sydent/db/accounts.py @@ -50,7 +50,7 @@ def getAccountByToken(self, token: str) -> Optional[Account]: return Account(*row) def storeAccount( - self, user_id: str, creation_ts: int, consent_version: Optional[bytes] + self, user_id: str, creation_ts: int, consent_version: Optional[str] ) -> None: """ Stores an account for the given user ID. @@ -72,7 +72,7 @@ def storeAccount( self.sydent.db.commit() def setConsentVersion( - self, user_id: bytes, consent_version: Optional[str] + self, user_id: str, consent_version: Optional[str] ) -> None: """ Saves that the given user has agreed to all of the terms in the document of the diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index 40b2024b..1a9dcabc 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -84,7 +84,7 @@ def store_lookup_pepper( def _rehash_threepids( self, - cur: "Cursor", + cur: Cursor, hashing_function: Callable[[str], str], pepper: str, table: str, diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index b9e39766..8ad31cee 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -61,7 +61,7 @@ def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: self.sydent.db.commit() def getAssociationsAfterId( - self, afterId: int, limit: Optional[int] = None + self, afterId: Optional[int], limit: Optional[int] = None ) -> Tuple[Dict[int, ThreepidAssociation], Optional[int]]: """ Retrieves every association after the given ID. @@ -106,7 +106,7 @@ def getAssociationsAfterId( return assocs, maxId def getSignedAssociationsAfterId( - self, afterId: int, limit: Optional[int] = None + self, afterId: Optional[int], limit: Optional[int] = None ) -> Tuple[Dict[int, Dict[str, Any]], Optional[int]]: """Get associations after a given ID, and sign them before returning @@ -254,7 +254,7 @@ def getMxid(self, medium: str, address: str) -> Optional[str]: return row[0] def getMxids( - self, threepid_tuples: List[Tuple[str, str, str]] + self, threepid_tuples: List[Tuple[str, str]] ) -> List[Tuple[str, str, str]]: """Given a list of threepid_tuples, return the same list but with mxids appended to each tuple for which a match was found in the @@ -295,7 +295,7 @@ def getMxids( ) results = [] - current = (Any, Any) + current = None for row in res.fetchall(): # only use the most recent entry for each # threepid (they're sorted by ts) @@ -401,7 +401,7 @@ def removeAssociation(self, medium: str, address: str) -> None: ) self.sydent.db.commit() - def retrieveMxidsForHashes(self, addresses: List[Any]) -> Dict[str, str]: + def retrieveMxidsForHashes(self, addresses: List[str]) -> Dict[str, str]: """Returns a mapping from hash: mxid from a list of given lookup_hash values :param addresses: An array of lookup_hash values to check against the db diff --git a/sydent/db/valsession.py b/sydent/db/valsession.py index b8afb227..423060f3 100644 --- a/sydent/db/valsession.py +++ b/sydent/db/valsession.py @@ -131,7 +131,7 @@ def addValSession( self.sydent.db.commit() return sid - def setSendAttemptNumber(self, sid: str, attemptNo: int) -> None: + def setSendAttemptNumber(self, sid: int, attemptNo: int) -> None: """ Updates the send attempt number for the session with the given ID. @@ -148,7 +148,7 @@ def setSendAttemptNumber(self, sid: str, attemptNo: int) -> None: ) self.sydent.db.commit() - def setValidated(self, sid: str, validated: bool) -> None: + def setValidated(self, sid: int, validated: bool) -> None: """ Updates a session to set the validated flag to the given value. @@ -165,7 +165,7 @@ def setValidated(self, sid: str, validated: bool) -> None: ) self.sydent.db.commit() - def setMtime(self, sid: str, mtime: int) -> None: + def setMtime(self, sid: int, mtime: int) -> None: """ Set the time of the last send attempt for the session with the given ID @@ -182,7 +182,7 @@ def setMtime(self, sid: str, mtime: int) -> None: ) self.sydent.db.commit() - def getSessionById(self, sid: str) -> Optional[ValidationSession]: + def getSessionById(self, sid: int) -> Optional[ValidationSession]: """ Retrieves the session matching the given sid. @@ -209,7 +209,7 @@ def getSessionById(self, sid: str) -> Optional[ValidationSession]: row[0], row[1], row[2], row[3], row[4], row[5], None, None ) - def getTokenSessionById(self, sid: str) -> Optional[ValidationSession]: + def getTokenSessionById(self, sid: int) -> Optional[ValidationSession]: """ Retrieves a validation session using the session's ID. @@ -237,7 +237,7 @@ def getTokenSessionById(self, sid: str) -> Optional[ValidationSession]: return None - def getValidatedSession(self, sid: str, clientSecret: str) -> ValidationSession: + def getValidatedSession(self, sid: int, clientSecret: str) -> ValidationSession: """ Retrieve a validated and still-valid session whose client secret matches the one passed in. diff --git a/sydent/hs_federation/verifier.py b/sydent/hs_federation/verifier.py index 146f84a8..abffa1a8 100644 --- a/sydent/hs_federation/verifier.py +++ b/sydent/hs_federation/verifier.py @@ -68,7 +68,7 @@ def __init__(self, sydent: 'Sydent') -> None: self.sydent = sydent # Cache of server keys. These are cached until the 'valid_until_ts' time # in the result. - self.cache: Dict = { + self.cache: Dict[str, Any] = { # server_name: , } diff --git a/sydent/http/auth.py b/sydent/http/auth.py index 16825a23..32a5f5f6 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -63,7 +63,7 @@ def authV2( sydent: "Sydent", request: 'Request', requireTermsAgreed: bool = True, -) -> Optional['Account']: +) -> 'Account': """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py index a339ff28..a2b0dab2 100644 --- a/sydent/http/httpclient.py +++ b/sydent/http/httpclient.py @@ -43,7 +43,7 @@ class HTTPClient(object): requests. """ - agent: Any # TODO: find type that will work with both Agent() and MatrixFederationAgent() + agent: IAgent # TODO: find type that will work with both Agent() and MatrixFederationAgent() @defer.inlineCallbacks def get_json( diff --git a/sydent/http/httpsclient.py b/sydent/http/httpsclient.py index f1f894b2..4cba20d0 100644 --- a/sydent/http/httpsclient.py +++ b/sydent/http/httpsclient.py @@ -20,9 +20,10 @@ from io import BytesIO from twisted.internet.ssl import optionsForClientTLS +from twisted.internet.defer import Deferred from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers -from twisted.web.iweb import IPolicyForHTTPS +from twisted.web.iweb import IPolicyForHTTPS, IResponse from zope.interface import implementer from typing import TYPE_CHECKING, Dict, Any, Generator, Optional @@ -52,7 +53,7 @@ def __init__(self, sydent: 'Sydent') -> None: # trustRoot=self.sydent.sslComponents.trustRoot) self.agent = Agent(self.sydent.reactor, SydentPolicyForHTTPS(self.sydent)) - def postJson(self, uri: str, jsonObject: Dict[Any, Any]) -> Optional[Generator]: + def postJson(self, uri: str, jsonObject: Dict[Any, Any]) -> Optional[Deferred]: """ Sends an POST request over HTTPS. diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py index 9d32dcfd..d32911ab 100644 --- a/sydent/http/matrixfederationagent.py +++ b/sydent/http/matrixfederationagent.py @@ -126,7 +126,7 @@ def request( method: bytes, uri: bytes, headers: Optional['Headers']=None, - bodyProducer: Optional['IBodyProducer']=None) -> 'Deferred': + bodyProducer: Optional['IBodyProducer']=None) -> Generator: """ :param method: HTTP method (GET/POST/etc). :type method: bytes @@ -304,7 +304,7 @@ def _route_matrix_uri(self, parsed_uri: 'URI', lookup_well_known: bool=True) -> ) @defer.inlineCallbacks - def _get_well_known(self, server_name: bytes) -> 'Deferred': + def _get_well_known(self, server_name: bytes) -> Generator: """Attempt to fetch and parse a .well-known file for the given server :param server_name: Name of the server, from the requested url. @@ -327,7 +327,7 @@ def _get_well_known(self, server_name: bytes) -> 'Deferred': defer.returnValue(result) @defer.inlineCallbacks - def _do_get_well_known(self, server_name: bytes) -> 'Deferred': + def _do_get_well_known(self, server_name: bytes) -> Generator: """Actually fetch and parse a .well-known, without checking the cache :param server_name: Name of the server, from the requested url diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index a6ca1dd5..0f30696d 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -26,7 +26,7 @@ from twisted.names import client, dns from twisted.names.error import DNSNameError, DomainError -from typing import Callable, Dict, Tuple, List +from typing import Callable, Dict, Tuple, List, SupportsInt logger = logging.getLogger(__name__) @@ -100,13 +100,13 @@ class SrvResolver(object): :type get_time: callable """ - def __init__(self, dns_client: 'IResolver'=client, cache: Dict=SERVER_CACHE, get_time: Callable=time.time) -> None: + def __init__(self, dns_client: 'IResolver'=client, cache: Dict[bytes, List[Server]]=SERVER_CACHE, get_time: Callable[[], SupportsInt]=time.time) -> None: self._dns_client = dns_client self._cache = cache self._get_time = get_time @defer.inlineCallbacks - def resolve_service(self, service_name: bytes) -> 'Deferred': + def resolve_service(self, service_name: bytes) -> Generator: """Look up a SRV record :param service_name: The record to look up. diff --git a/sydent/replication/pusher.py b/sydent/replication/pusher.py index 1b12df5e..ccff5186 100644 --- a/sydent/replication/pusher.py +++ b/sydent/replication/pusher.py @@ -27,7 +27,7 @@ from sydent.db.threepid_associations import LocalAssociationStore from sydent.db.peers import PeerStore -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Generator if TYPE_CHECKING: from sydent.sydent import Sydent @@ -79,7 +79,7 @@ def scheduledPush(self): return defer.DeferredList([self._push_to_peer(p) for p in peers]) @defer.inlineCallbacks - def _push_to_peer(self, p: 'RemotePeer') -> None: + def _push_to_peer(self, p: 'RemotePeer') -> Generator: """ For a given peer, retrieves the list of associations that were created since the last successful push to this peer (limited to ASSOCIATIONS_PUSH_LIMIT) and diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index 4a2b2774..b746d8c5 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -140,4 +140,4 @@ def get_terms(sydent) -> Optional[Terms]: logger.exception( "Couldn't read terms file '%s'", sydent.cfg.get("general", "terms.path") ) - return None # added per this: https://github.com/python/mypy/issues/3974 + return None diff --git a/sydent/util/hash.py b/sydent/util/hash.py index 73328dce..70df3a4e 100644 --- a/sydent/util/hash.py +++ b/sydent/util/hash.py @@ -19,7 +19,7 @@ import unpaddedbase64 # type: ignore -def sha256_and_url_safe_base64(input_text: str) -> bytes: +def sha256_and_url_safe_base64(input_text: str) -> str: """SHA256 hash an input string, encode the digest as url-safe base64, and return diff --git a/sydent/validators/msisdnvalidator.py b/sydent/validators/msisdnvalidator.py index 3dbf1220..4596525a 100644 --- a/sydent/validators/msisdnvalidator.py +++ b/sydent/validators/msisdnvalidator.py @@ -39,7 +39,7 @@ def __init__(self, sydent: "Sydent") -> None: self.omSms = OpenMarketSMS(sydent) # cache originators & sms rules from config file - self.originators: Dict = {} + self.originators = {} self.smsRules = {} for opt in self.sydent.cfg.options("sms"): if opt.startswith("originators."): From c5cce3872eb226d5a24254a39ce87e87f2dafdac Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 8 Jun 2021 09:55:21 -0700 Subject: [PATCH 10/14] implement requested changes on PR --- sydent/db/threepid_associations.py | 6 +++--- sydent/http/auth.py | 3 +-- sydent/http/httpclient.py | 2 +- sydent/http/srvresolver.py | 4 ++-- sydent/sms/openmarket.py | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index 8ad31cee..fad2cc77 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -424,14 +424,14 @@ def retrieveMxidsForHashes(self, addresses: List[str]) -> Dict[str, str]: results = {} try: # Convert list of addresses to list of tuples of addresses - addresses = [(x,) for x in addresses] + tuplized_addresses = [(x,) for x in addresses] inserted_cap = 0 - while inserted_cap < len(addresses): + while inserted_cap < len(tuplized_addresses): cur.executemany( "INSERT INTO tmp_retrieve_mxids_for_hashes(lookup_hash) " "VALUES (?)", - addresses[inserted_cap : inserted_cap + 500], + tuplized_addresses[inserted_cap : inserted_cap + 500], ) inserted_cap += 500 diff --git a/sydent/http/auth.py b/sydent/http/auth.py index 32a5f5f6..d15f912a 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -73,8 +73,7 @@ def authV2( :param requireTermsAgreed: Whether to deny authentication if the user hasn't accepted the terms of service. - :returns Account|None: The account object if there is correct auth, or None for v1 - APIs. + :returns Account: The account object if there is correct auth :raises MatrixRestError: If the request is v2 but could not be authed or the user has not accepted terms. """ diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py index a2b0dab2..52edd634 100644 --- a/sydent/http/httpclient.py +++ b/sydent/http/httpclient.py @@ -43,7 +43,7 @@ class HTTPClient(object): requests. """ - agent: IAgent # TODO: find type that will work with both Agent() and MatrixFederationAgent() + agent: IAgent @defer.inlineCallbacks def get_json( diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index 0f30696d..8384ca78 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -26,7 +26,7 @@ from twisted.names import client, dns from twisted.names.error import DNSNameError, DomainError -from typing import Callable, Dict, Tuple, List, SupportsInt +from typing import Callable, Dict, Tuple, List, SupportsInt, Generator logger = logging.getLogger(__name__) @@ -106,7 +106,7 @@ def __init__(self, dns_client: 'IResolver'=client, cache: Dict[bytes, List[Serve self._get_time = get_time @defer.inlineCallbacks - def resolve_service(self, service_name: bytes) -> Generator: + def resolve_service(self, service_name: bytes) -> 'Generator': """Look up a SRV record :param service_name: The record to look up. diff --git a/sydent/sms/openmarket.py b/sydent/sms/openmarket.py index bcd46ae9..60aea3c5 100644 --- a/sydent/sms/openmarket.py +++ b/sydent/sms/openmarket.py @@ -99,7 +99,7 @@ def sendTextSMS( password = self.sydent.cfg.get("sms", "password").encode("UTF-8") b64creds = b64encode(b"%s:%s" % (username, password)) - headers: Any = Headers( + req_headers = Headers( { b"Authorization": [b"Basic " + b64creds], b"Content-Type": [b"application/json"], @@ -107,7 +107,7 @@ def sendTextSMS( ) resp = yield self.http_cli.post_json_get_nothing( - API_BASE_URL, body, {"headers": headers} + API_BASE_URL, body, {"headers": req_headers} ) headers = dict(resp.headers.getAllRawHeaders()) From d7bde1bc3056550627b6f83136aa0b495247501e Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 8 Jun 2021 09:57:12 -0700 Subject: [PATCH 11/14] reformat files with black and isort to pass lints --- sydent/db/accounts.py | 11 +++--- sydent/db/hashing_metadata.py | 5 ++- sydent/db/invite_tokens.py | 5 ++- sydent/db/peers.py | 6 ++-- sydent/db/sqlitedb.py | 3 +- sydent/db/terms.py | 4 +-- sydent/db/threepid_associations.py | 9 ++--- sydent/db/valsession.py | 5 ++- sydent/hs_federation/verifier.py | 18 ++++------ sydent/http/auth.py | 13 ++++--- sydent/http/httpclient.py | 11 +++--- sydent/http/httpcommon.py | 6 ++-- sydent/http/httpsclient.py | 9 +++-- sydent/http/httpserver.py | 9 +++-- sydent/http/matrixfederationagent.py | 22 ++++++------ sydent/http/srvresolver.py | 12 ++++--- sydent/replication/peer.py | 51 ++++++++++++---------------- sydent/replication/pusher.py | 13 ++++--- sydent/sms/openmarket.py | 6 ++-- sydent/terms/terms.py | 2 +- sydent/threepid/bind.py | 9 ++--- sydent/threepid/signer.py | 4 +-- sydent/users/tokens.py | 3 +- sydent/util/emailutils.py | 4 +-- sydent/util/ttlcache.py | 3 +- sydent/validators/common.py | 3 +- sydent/validators/emailvalidator.py | 5 +-- sydent/validators/msisdnvalidator.py | 3 +- 28 files changed, 107 insertions(+), 147 deletions(-) diff --git a/sydent/db/accounts.py b/sydent/db/accounts.py index 68f0a522..d761082d 100644 --- a/sydent/db/accounts.py +++ b/sydent/db/accounts.py @@ -15,15 +15,16 @@ # limitations under the License. from __future__ import absolute_import -from sydent.users.accounts import Account +from typing import TYPE_CHECKING, Optional -from typing import Optional, TYPE_CHECKING +from sydent.users.accounts import Account if TYPE_CHECKING: from sydent.sydent import Sydent + class AccountStore(object): - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getAccountByToken(self, token: str) -> Optional[Account]: @@ -71,9 +72,7 @@ def storeAccount( ) self.sydent.db.commit() - def setConsentVersion( - self, user_id: str, consent_version: Optional[str] - ) -> None: + def setConsentVersion(self, user_id: str, consent_version: Optional[str]) -> None: """ Saves that the given user has agreed to all of the terms in the document of the given version. diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index 1a9dcabc..22480ae9 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -17,15 +17,14 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py from sqlite3 import Cursor - -from typing import Union, Callable, TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Callable, Optional, Union if TYPE_CHECKING: from sydent.sydent import Sydent class HashingMetadataStore: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def get_lookup_pepper(self) -> Optional[str]: diff --git a/sydent/db/invite_tokens.py b/sydent/db/invite_tokens.py index fcc9755b..f419f286 100644 --- a/sydent/db/invite_tokens.py +++ b/sydent/db/invite_tokens.py @@ -14,15 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. import time - -from typing import Optional, List, Dict, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, List, Optional if TYPE_CHECKING: from sydent.sydent import Sydent class JoinTokenStore(object): - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def storeToken( diff --git a/sydent/db/peers.py b/sydent/db/peers.py index 4ba9b176..64137983 100644 --- a/sydent/db/peers.py +++ b/sydent/db/peers.py @@ -15,16 +15,16 @@ # limitations under the License. from __future__ import absolute_import -from sydent.replication.peer import RemotePeer +from typing import TYPE_CHECKING, Dict, List, Optional -from typing import Optional, List, Dict, TYPE_CHECKING +from sydent.replication.peer import RemotePeer if TYPE_CHECKING: from sydent.sydent import Sydent class PeerStore: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getPeerByName(self, name: str) -> Optional[RemotePeer]: diff --git a/sydent/db/sqlitedb.py b/sydent/db/sqlitedb.py index 46e820ac..9c2fa3f1 100644 --- a/sydent/db/sqlitedb.py +++ b/sydent/db/sqlitedb.py @@ -18,7 +18,6 @@ import logging import os import sqlite3 - from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -28,7 +27,7 @@ class SqliteDatabase: - def __init__(self, syd: 'Sydent') -> None: + def __init__(self, syd: "Sydent") -> None: self.sydent = syd dbFilePath = self.sydent.cfg.get("db", "db.file") diff --git a/sydent/db/terms.py b/sydent/db/terms.py index d7682697..99e15c42 100644 --- a/sydent/db/terms.py +++ b/sydent/db/terms.py @@ -14,14 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING, List if TYPE_CHECKING: from sydent.sydent import Sydent class TermsStore(object): - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def getAgreedUrls(self, user_id: str) -> List[str]: diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index fad2cc77..69121265 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -16,14 +16,11 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer -import logging - -from typing import Tuple, Optional, List, Dict, Any, Sequence, TYPE_CHECKING - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -31,7 +28,7 @@ class LocalAssociationStore: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def addOrUpdateAssociation(self, assoc: ThreepidAssociation) -> None: @@ -186,7 +183,7 @@ def removeAssociation(self, threepid: Dict[str, str], mxid: str) -> None: class GlobalAssociationStore: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def signedAssociationStringForThreepid( diff --git a/sydent/db/valsession.py b/sydent/db/valsession.py index 423060f3..aba15322 100644 --- a/sydent/db/valsession.py +++ b/sydent/db/valsession.py @@ -16,6 +16,7 @@ from __future__ import absolute_import from random import SystemRandom +from typing import TYPE_CHECKING, Optional import sydent.util.tokenutils from sydent.util import time_msec @@ -27,14 +28,12 @@ ValidationSession, ) -from typing import TYPE_CHECKING, Optional - if TYPE_CHECKING: from sydent.sydent import Sydent class ThreePidValSessionStore: - def __init__(self, syd: 'Sydent') -> None: + def __init__(self, syd: "Sydent") -> None: self.sydent = syd self.random = SystemRandom() diff --git a/sydent/hs_federation/verifier.py b/sydent/hs_federation/verifier.py index abffa1a8..2dfb0a28 100644 --- a/sydent/hs_federation/verifier.py +++ b/sydent/hs_federation/verifier.py @@ -17,24 +17,18 @@ import logging import time +from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple - -import signedjson.key -import signedjson.sign -from twisted.internet import defer -from twisted.web.server import Request -from unpaddedbase64 import decode_base64 # type: ignore -import signedjson.sign # type: ignore import signedjson.key # type: ignore +import signedjson.sign # type: ignore from signedjson.sign import SignatureVerifyException from twisted.internet import defer -from unpaddedbase64 import decode_base64 +from twisted.web.server import Request +from unpaddedbase64 import decode_base64 # type: ignore from sydent.http.httpclient import FederationHttpClient from sydent.util.stringutils import is_valid_matrix_server_name -from typing import Dict, Optional, TYPE_CHECKING, Any, List, Tuple, Generator - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -64,7 +58,7 @@ class Verifier(object): verifying that the signature on the json blob matches. """ - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent # Cache of server keys. These are cached until the 'valid_until_ts' time # in the result. @@ -178,7 +172,7 @@ def verifyServerSignedJson( @defer.inlineCallbacks def authenticate_request( - self, request: 'Request', content: Optional[bytes] + self, request: "Request", content: Optional[bytes] ) -> Generator: """Authenticates a Matrix federation request based on the X-Matrix header XXX: Copied largely from synapse diff --git a/sydent/http/auth.py b/sydent/http/auth.py index d15f912a..d4e78066 100644 --- a/sydent/http/auth.py +++ b/sydent/http/auth.py @@ -16,15 +16,14 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Optional + +from twisted.web.server import Request from sydent.db.accounts import AccountStore from sydent.http.servlets import MatrixRestError, get_args from sydent.terms.terms import get_terms -from twisted.web.server import Request - -from typing import Optional, TYPE_CHECKING - if TYPE_CHECKING: from sydent.db.accounts import Account from sydent.sydent import Sydent @@ -32,7 +31,7 @@ logger = logging.getLogger(__name__) -def tokenFromRequest(request: 'Request') -> Optional[str]: +def tokenFromRequest(request: "Request") -> Optional[str]: """Extract token from header of query parameter. :param request: The request to look for an access token in. @@ -61,9 +60,9 @@ def tokenFromRequest(request: 'Request') -> Optional[str]: def authV2( sydent: "Sydent", - request: 'Request', + request: "Request", requireTermsAgreed: bool = True, -) -> 'Account': +) -> "Account": """For v2 APIs check that the request has a valid access token associated with it :param sydent: The Sydent instance to use. diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py index 52edd634..bfdcf2bc 100644 --- a/sydent/http/httpclient.py +++ b/sydent/http/httpclient.py @@ -18,6 +18,7 @@ import json import logging from io import BytesIO +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional from twisted.internet import defer from twisted.web.client import Agent, FileBodyProducer @@ -30,8 +31,6 @@ from sydent.http.matrixfederationagent import MatrixFederationAgent from sydent.util import json_decoder -from typing import Optional, Dict, Any, TYPE_CHECKING, Generator - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -46,9 +45,7 @@ class HTTPClient(object): agent: IAgent @defer.inlineCallbacks - def get_json( - self, uri: str, max_size: Optional[int] = None - ) -> Generator: + def get_json(self, uri: str, max_size: Optional[int] = None) -> Generator: """Make a GET request to an endpoint returning JSON and parse result :param uri: The URI to make a GET request to. @@ -132,7 +129,7 @@ class SimpleHttpClient(HTTPClient): from Synapse. """ - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent # The default endpoint factory in Twisted 14.0.0 (which we require) uses the # BrowserLikePolicyForHTTPS context factory which will do regular cert validation @@ -152,7 +149,7 @@ class FederationHttpClient(HTTPClient): MatrixFederationAgent. """ - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.agent = MatrixFederationAgent( BlacklistingReactorWrapper( diff --git a/sydent/http/httpcommon.py b/sydent/http/httpcommon.py index cea224ff..6d595d4b 100644 --- a/sydent/http/httpcommon.py +++ b/sydent/http/httpcommon.py @@ -16,6 +16,7 @@ import logging from io import BytesIO +from typing import TYPE_CHECKING import twisted.internet.ssl from twisted.internet import defer, protocol @@ -24,9 +25,6 @@ from twisted.web._newclient import ResponseDone from twisted.web.http import PotentialDataLoss from twisted.web.iweb import UNKNOWN_LENGTH -from twisted.web import server - -from typing import TYPE_CHECKING if TYPE_CHECKING: from sydent.sydent import Sydent @@ -39,7 +37,7 @@ class SslComponents: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.myPrivateCertificate = self.makeMyCertificate() diff --git a/sydent/http/httpsclient.py b/sydent/http/httpsclient.py index 4cba20d0..217e6feb 100644 --- a/sydent/http/httpsclient.py +++ b/sydent/http/httpsclient.py @@ -18,16 +18,15 @@ import json import logging from io import BytesIO +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional -from twisted.internet.ssl import optionsForClientTLS from twisted.internet.defer import Deferred +from twisted.internet.ssl import optionsForClientTLS from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers from twisted.web.iweb import IPolicyForHTTPS, IResponse from zope.interface import implementer -from typing import TYPE_CHECKING, Dict, Any, Generator, Optional - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -41,7 +40,7 @@ class ReplicationHttpsClient: replication HTTPS server) """ - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.agent = None @@ -84,7 +83,7 @@ def postJson(self, uri: str, jsonObject: Dict[Any, Any]) -> Optional[Deferred]: @implementer(IPolicyForHTTPS) class SydentPolicyForHTTPS(object): - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def creatorForNetloc(self, hostname, port): diff --git a/sydent/http/httpserver.py b/sydent/http/httpserver.py index 640254a4..1b4e3bb0 100644 --- a/sydent/http/httpserver.py +++ b/sydent/http/httpserver.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING import twisted.internet.ssl from twisted.web.resource import Resource @@ -31,8 +32,6 @@ AuthenticatedUnbindThreePidServlet, ) -from typing import TYPE_CHECKING - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -40,7 +39,7 @@ class ClientApiHttpServer: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent root = Resource() @@ -154,7 +153,7 @@ def setup(self): class InternalApiHttpServer(object): - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent def setup(self, interface, port): @@ -182,7 +181,7 @@ def setup(self, interface, port): class ReplicationHttpsServer: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent root = Resource() diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py index d32911ab..45540012 100644 --- a/sydent/http/matrixfederationagent.py +++ b/sydent/http/matrixfederationagent.py @@ -17,11 +17,10 @@ import logging import random import time +from typing import Generator, Optional import attr -from netaddr import IPAddress # type: ignore -from zope.interface import implementer - +from netaddr import IPAddress # type: ignore from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS @@ -30,15 +29,13 @@ from twisted.web.http import stringToDatetime from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent, IBodyProducer - +from zope.interface import implementer from sydent.http.httpcommon import read_body_with_max_size from sydent.http.srvresolver import SrvResolver, pick_server_from_list from sydent.util import json_decoder from sydent.util.ttlcache import TTLCache -from typing import Generator, Optional - # period to cache .well-known results for by default WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600 @@ -89,8 +86,8 @@ def __init__( reactor, tls_client_options_factory, _well_known_tls_policy=None, - _srv_resolver: Optional['SrvResolver']=None, - _well_known_cache: Optional['TTLCache']=well_known_cache, + _srv_resolver: Optional["SrvResolver"] = None, + _well_known_cache: Optional["TTLCache"] = well_known_cache, ) -> None: self._reactor = reactor @@ -125,8 +122,9 @@ def request( self, method: bytes, uri: bytes, - headers: Optional['Headers']=None, - bodyProducer: Optional['IBodyProducer']=None) -> Generator: + headers: Optional["Headers"] = None, + bodyProducer: Optional["IBodyProducer"] = None, + ) -> Generator: """ :param method: HTTP method (GET/POST/etc). :type method: bytes @@ -191,7 +189,9 @@ def endpointForURI(_uri): defer.returnValue(res) @defer.inlineCallbacks - def _route_matrix_uri(self, parsed_uri: 'URI', lookup_well_known: bool=True) -> 'Deferred': + def _route_matrix_uri( + self, parsed_uri: "URI", lookup_well_known: bool = True + ) -> "Deferred": """Helper for `request`: determine the routing for a Matrix URI :param parsed_uri: uri to route. Note that it should be parsed with diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index 8384ca78..13b4a335 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -17,6 +17,7 @@ import logging import random import time +from typing import Callable, Dict, Generator, List, SupportsInt, Tuple import attr from twisted.internet import defer @@ -26,8 +27,6 @@ from twisted.names import client, dns from twisted.names.error import DNSNameError, DomainError -from typing import Callable, Dict, Tuple, List, SupportsInt, Generator - logger = logging.getLogger(__name__) SERVER_CACHE = {} @@ -100,13 +99,18 @@ class SrvResolver(object): :type get_time: callable """ - def __init__(self, dns_client: 'IResolver'=client, cache: Dict[bytes, List[Server]]=SERVER_CACHE, get_time: Callable[[], SupportsInt]=time.time) -> None: + def __init__( + self, + dns_client: "IResolver" = client, + cache: Dict[bytes, List[Server]] = SERVER_CACHE, + get_time: Callable[[], SupportsInt] = time.time, + ) -> None: self._dns_client = dns_client self._cache = cache self._get_time = get_time @defer.inlineCallbacks - def resolve_service(self, service_name: bytes) -> 'Generator': + def resolve_service(self, service_name: bytes) -> "Generator": """Look up a SRV record :param service_name: The record to look up. diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py index ff103555..ca5dfde5 100644 --- a/sydent/replication/peer.py +++ b/sydent/replication/peer.py @@ -19,13 +19,16 @@ import binascii import json import logging +from typing import TYPE_CHECKING, Any, Dict, Generator -import signedjson.key -import signedjson.sign +import signedjson.key # type: ignore +import signedjson.sign # type: ignore from six.moves import configparser from twisted.internet import defer +from twisted.internet.defer import Deferred from twisted.web.client import readBody -from unpaddedbase64 import decode_base64 +from twisted.web.iweb import IResponse +from unpaddedbase64 import decode_base64 # type: ignore from sydent.config import ConfigError from sydent.db.hashing_metadata import HashingMetadataStore @@ -33,21 +36,6 @@ from sydent.threepid import threePidAssocFromDict from sydent.util import json_decoder from sydent.util.hash import sha256_and_url_safe_base64 -from unpaddedbase64 import decode_base64 # type: ignore - -import signedjson.sign # type: ignore -import signedjson.key # type: ignore - -import logging -import json -import binascii - -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.web.client import readBody -from twisted.web.iweb import IResponse - -from typing import Generator, Dict, Any, TYPE_CHECKING if TYPE_CHECKING: from sydent.sydent import Sydent @@ -63,7 +51,7 @@ def __init__(self, servername, pubkeys): self.pubkeys = pubkeys self.is_being_pushed_to = False - def pushUpdates(self, sgAssocs) -> 'Deferred': + def pushUpdates(self, sgAssocs) -> "Deferred": """ :param sgAssocs: Sequence of (originId, sgAssoc) tuples where originId is the id on the creating server and sgAssoc is the json object of the signed association @@ -77,7 +65,7 @@ class LocalPeer(Peer): The local peer (ourselves: essentially copying from the local associations table to the global one) """ - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: super(LocalPeer, self).__init__(sydent.server_name, {}) self.sydent = sydent self.hashing_store = HashingMetadataStore(sydent) @@ -87,7 +75,7 @@ def __init__(self, sydent: 'Sydent') -> None: if self.lastId is None: self.lastId = -1 - def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> "Deferred": """ Saves the given associations in the global associations store. Only stores an association if its ID is greater than the last seen ID. @@ -132,7 +120,14 @@ def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': class RemotePeer(Peer): - def __init__(self, sydent: 'Sydent', server_name: str, port: int, pubkeys: Dict[str, str], lastSentVersion: int) -> None: + def __init__( + self, + sydent: "Sydent", + server_name: str, + port: int, + pubkeys: Dict[str, str], + lastSentVersion: int, + ) -> None: """ :param sydent: The current Sydent instance. :type sydent: sydent.sydent.Sydent @@ -223,7 +218,7 @@ def verifySignedAssociation(self, assoc: Dict[Any, Any]) -> None: # Verify the JSON signedjson.sign.verify_signed_json(assoc, self.servername, self.verify_key) - def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': + def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> "Deferred": """ Pushes the given associations to the peer. @@ -253,8 +248,8 @@ def pushUpdates(self, sgAssocs: Dict[int, Dict[str, Any]]) -> 'Deferred': def _pushSuccess( self, - result: 'IResponse', - updateDeferred: 'Deferred', + result: "IResponse", + updateDeferred: "Deferred", ) -> None: """ Processes a successful push request. If the request resulted in a status code @@ -273,9 +268,7 @@ def _pushSuccess( d.addCallback(self._failedPushBodyRead, updateDeferred=updateDeferred) d.addErrback(self._pushFailed, updateDeferred=updateDeferred) - def _failedPushBodyRead( - self, body: bytes, updateDeferred: 'Deferred' - ) -> None: + def _failedPushBodyRead(self, body: bytes, updateDeferred: "Deferred") -> None: """ Processes a response body from a failed push request, then calls the error callback of the provided deferred. @@ -293,7 +286,7 @@ def _failedPushBodyRead( def _pushFailed( self, failure, - updateDeferred: 'Deferred', + updateDeferred: "Deferred", ) -> None: """ Processes a failed push request, by calling the error callback of the given diff --git a/sydent/replication/pusher.py b/sydent/replication/pusher.py index ccff5186..af8ab0b4 100644 --- a/sydent/replication/pusher.py +++ b/sydent/replication/pusher.py @@ -17,17 +17,16 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Generator import twisted.internet.reactor import twisted.internet.task from twisted.internet import defer -from sydent.util import time_msec -from sydent.replication.peer import LocalPeer, RemotePeer -from sydent.db.threepid_associations import LocalAssociationStore from sydent.db.peers import PeerStore - -from typing import TYPE_CHECKING, Generator +from sydent.db.threepid_associations import LocalAssociationStore +from sydent.replication.peer import LocalPeer, RemotePeer +from sydent.util import time_msec if TYPE_CHECKING: from sydent.sydent import Sydent @@ -39,7 +38,7 @@ class Pusher: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.pushing = False self.peerStore = PeerStore(self.sydent) @@ -79,7 +78,7 @@ def scheduledPush(self): return defer.DeferredList([self._push_to_peer(p) for p in peers]) @defer.inlineCallbacks - def _push_to_peer(self, p: 'RemotePeer') -> Generator: + def _push_to_peer(self, p: "RemotePeer") -> Generator: """ For a given peer, retrieves the list of associations that were created since the last successful push to this peer (limited to ASSOCIATIONS_PUSH_LIMIT) and diff --git a/sydent/sms/openmarket.py b/sydent/sms/openmarket.py index 60aea3c5..ccecbaad 100644 --- a/sydent/sms/openmarket.py +++ b/sydent/sms/openmarket.py @@ -17,15 +17,13 @@ import logging from base64 import b64encode +from typing import TYPE_CHECKING, Any, Dict, Optional from twisted.internet import defer from twisted.web.http_headers import Headers - from sydent.http.httpclient import SimpleHttpClient -from typing import Dict, Any, TYPE_CHECKING, Optional - if TYPE_CHECKING: from sydent.sydent import Sydent @@ -62,7 +60,7 @@ def tonFromType(t: str) -> int: class OpenMarketSMS: - def __init__(self, sydent: 'Sydent') -> None: + def __init__(self, sydent: "Sydent") -> None: self.sydent = sydent self.http_cli = SimpleHttpClient(sydent) diff --git a/sydent/terms/terms.py b/sydent/terms/terms.py index b746d8c5..0568a2b8 100644 --- a/sydent/terms/terms.py +++ b/sydent/terms/terms.py @@ -15,9 +15,9 @@ # limitations under the License. import logging +from typing import Any, Dict, List, Optional, Set import yaml -from typing import Any, Set, List, Dict, Optional logger = logging.getLogger(__name__) diff --git a/sydent/threepid/bind.py b/sydent/threepid/bind.py index da68f740..4417f8e6 100644 --- a/sydent/threepid/bind.py +++ b/sydent/threepid/bind.py @@ -19,10 +19,9 @@ import collections import logging import math -import signedjson.sign # type: ignore -from sydent.db.invite_tokens import JoinTokenStore +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Union -import signedjson.sign +import signedjson.sign # type: ignore from twisted.internet import defer from sydent.db.hashing_metadata import HashingMetadataStore @@ -35,10 +34,6 @@ from sydent.util.hash import sha256_and_url_safe_base64 from sydent.util.stringutils import is_valid_matrix_server_name -from twisted.internet import defer - -from typing import TYPE_CHECKING, Dict, Any, Generator, Union, Optional - if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/threepid/signer.py b/sydent/threepid/signer.py index 78101cc5..1f03cfe1 100644 --- a/sydent/threepid/signer.py +++ b/sydent/threepid/signer.py @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import signedjson.sign # type: ignore +from typing import TYPE_CHECKING, Any, Dict -from typing import TYPE_CHECKING, Dict, Any +import signedjson.sign # type: ignore if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/users/tokens.py b/sydent/users/tokens.py index 0bc99dca..5e9cc53f 100644 --- a/sydent/users/tokens.py +++ b/sydent/users/tokens.py @@ -16,12 +16,11 @@ import logging import time +from typing import TYPE_CHECKING from sydent.db.accounts import AccountStore from sydent.util.tokenutils import generateAlphanumericTokenOfLength -from typing import TYPE_CHECKING - if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/util/emailutils.py b/sydent/util/emailutils.py index 72a8dbcc..844c5772 100644 --- a/sydent/util/emailutils.py +++ b/sydent/util/emailutils.py @@ -31,11 +31,11 @@ else: from html import escape +from typing import TYPE_CHECKING, Any, Dict + from sydent.util import time_msec from sydent.util.tokenutils import generateAlphanumericTokenOfLength -from typing import TYPE_CHECKING, Dict, Any - if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/util/ttlcache.py b/sydent/util/ttlcache.py index b24232c8..fbcd9d49 100644 --- a/sydent/util/ttlcache.py +++ b/sydent/util/ttlcache.py @@ -15,12 +15,11 @@ import logging import time +from typing import Any, Dict, Tuple import attr from sortedcontainers import SortedList # type: ignore -from typing import Any, Dict, Tuple - logger = logging.getLogger(__name__) SENTINEL = object() diff --git a/sydent/validators/common.py b/sydent/validators/common.py index 49bf29de..31efa217 100644 --- a/sydent/validators/common.py +++ b/sydent/validators/common.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Dict from sydent.db.valsession import ThreePidValSessionStore from sydent.util import time_msec @@ -12,8 +13,6 @@ ValidationSession, ) -from typing import TYPE_CHECKING, Dict - if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/validators/emailvalidator.py b/sydent/validators/emailvalidator.py index 6ee292bc..d91b8e55 100644 --- a/sydent/validators/emailvalidator.py +++ b/sydent/validators/emailvalidator.py @@ -16,6 +16,7 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Dict, Optional from six.moves import urllib @@ -24,10 +25,6 @@ from sydent.util.emailutils import sendEmail from sydent.validators import common -from sydent.util import time_msec - -from typing import TYPE_CHECKING, Optional, Dict - if TYPE_CHECKING: from sydent.sydent import Sydent from sydent.validators import ValidationSession diff --git a/sydent/validators/msisdnvalidator.py b/sydent/validators/msisdnvalidator.py index 4596525a..b66c390b 100644 --- a/sydent/validators/msisdnvalidator.py +++ b/sydent/validators/msisdnvalidator.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import logging +from typing import TYPE_CHECKING, Dict, Optional import phonenumbers # type: ignore @@ -25,8 +26,6 @@ from sydent.util import time_msec from sydent.validators import DestinationRejectedException, common -from typing import TYPE_CHECKING, Optional, Dict - if TYPE_CHECKING: from sydent.sydent import Sydent From 1cd8b91a614b2f23571a5c9c470c5d8ece5e6966 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 8 Jun 2021 12:52:26 -0700 Subject: [PATCH 12/14] resolve requested changes on PR --- sydent/validators/msisdnvalidator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sydent/validators/msisdnvalidator.py b/sydent/validators/msisdnvalidator.py index b66c390b..6080d60a 100644 --- a/sydent/validators/msisdnvalidator.py +++ b/sydent/validators/msisdnvalidator.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import logging -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional, List import phonenumbers # type: ignore @@ -38,7 +38,7 @@ def __init__(self, sydent: "Sydent") -> None: self.omSms = OpenMarketSMS(sydent) # cache originators & sms rules from config file - self.originators = {} + self.originators: Dict[str, List[Dict[str, str]]] = {} self.smsRules = {} for opt in self.sydent.cfg.options("sms"): if opt.startswith("originators."): From 7b4bc8dbe39e23f9207539047ecde3a27fc96e7a Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 8 Jun 2021 15:09:19 -0700 Subject: [PATCH 13/14] fix flake8 lint errors --- sydent/db/hashing_metadata.py | 2 +- sydent/db/threepid_associations.py | 3 ++- sydent/http/httpsclient.py | 4 ++-- sydent/http/srvresolver.py | 1 - sydent/replication/peer.py | 2 +- sydent/sms/openmarket.py | 2 +- sydent/threepid/bind.py | 2 +- sydent/util/ttlcache.py | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sydent/db/hashing_metadata.py b/sydent/db/hashing_metadata.py index 22480ae9..0a70ef0b 100644 --- a/sydent/db/hashing_metadata.py +++ b/sydent/db/hashing_metadata.py @@ -17,7 +17,7 @@ # Actions on the hashing_metadata table which is defined in the migration process in # sqlitedb.py from sqlite3 import Cursor -from typing import TYPE_CHECKING, Callable, Optional, Union +from typing import TYPE_CHECKING, Callable, Optional if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index 69121265..94e5ebcd 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -16,8 +16,9 @@ from __future__ import absolute_import import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from sydent.util import time_msec from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer diff --git a/sydent/http/httpsclient.py b/sydent/http/httpsclient.py index 217e6feb..942a34d3 100644 --- a/sydent/http/httpsclient.py +++ b/sydent/http/httpsclient.py @@ -18,13 +18,13 @@ import json import logging from io import BytesIO -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from twisted.internet.defer import Deferred from twisted.internet.ssl import optionsForClientTLS from twisted.web.client import Agent, FileBodyProducer from twisted.web.http_headers import Headers -from twisted.web.iweb import IPolicyForHTTPS, IResponse +from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer if TYPE_CHECKING: diff --git a/sydent/http/srvresolver.py b/sydent/http/srvresolver.py index 13b4a335..db4a72e8 100644 --- a/sydent/http/srvresolver.py +++ b/sydent/http/srvresolver.py @@ -21,7 +21,6 @@ import attr from twisted.internet import defer -from twisted.internet.defer import Deferred from twisted.internet.error import ConnectError from twisted.internet.interfaces import IResolver from twisted.names import client, dns diff --git a/sydent/replication/peer.py b/sydent/replication/peer.py index ca5dfde5..827afab8 100644 --- a/sydent/replication/peer.py +++ b/sydent/replication/peer.py @@ -19,7 +19,7 @@ import binascii import json import logging -from typing import TYPE_CHECKING, Any, Dict, Generator +from typing import TYPE_CHECKING, Any, Dict import signedjson.key # type: ignore import signedjson.sign # type: ignore diff --git a/sydent/sms/openmarket.py b/sydent/sms/openmarket.py index ccecbaad..c7b11b88 100644 --- a/sydent/sms/openmarket.py +++ b/sydent/sms/openmarket.py @@ -17,7 +17,7 @@ import logging from base64 import b64encode -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional from twisted.internet import defer from twisted.web.http_headers import Headers diff --git a/sydent/threepid/bind.py b/sydent/threepid/bind.py index 4417f8e6..51dee029 100644 --- a/sydent/threepid/bind.py +++ b/sydent/threepid/bind.py @@ -19,7 +19,7 @@ import collections import logging import math -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Union import signedjson.sign # type: ignore from twisted.internet import defer diff --git a/sydent/util/ttlcache.py b/sydent/util/ttlcache.py index fbcd9d49..702ef618 100644 --- a/sydent/util/ttlcache.py +++ b/sydent/util/ttlcache.py @@ -15,7 +15,7 @@ import logging import time -from typing import Any, Dict, Tuple +from typing import Any, Tuple import attr from sortedcontainers import SortedList # type: ignore From dbbeb1eeac4a8b0990465765a4eb9c6dfa3cbd65 Mon Sep 17 00:00:00 2001 From: "H.Shay" Date: Tue, 8 Jun 2021 15:15:59 -0700 Subject: [PATCH 14/14] fix isort lint fail --- sydent/db/threepid_associations.py | 2 +- sydent/validators/msisdnvalidator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sydent/db/threepid_associations.py b/sydent/db/threepid_associations.py index 94e5ebcd..48227d5d 100644 --- a/sydent/db/threepid_associations.py +++ b/sydent/db/threepid_associations.py @@ -18,9 +18,9 @@ import logging from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple -from sydent.util import time_msec from sydent.threepid import ThreepidAssociation from sydent.threepid.signer import Signer +from sydent.util import time_msec if TYPE_CHECKING: from sydent.sydent import Sydent diff --git a/sydent/validators/msisdnvalidator.py b/sydent/validators/msisdnvalidator.py index 6080d60a..c6ba57ac 100644 --- a/sydent/validators/msisdnvalidator.py +++ b/sydent/validators/msisdnvalidator.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import logging -from typing import TYPE_CHECKING, Dict, Optional, List +from typing import TYPE_CHECKING, Dict, List, Optional import phonenumbers # type: ignore