Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Federation outbound proxy #15773

Merged
merged 47 commits into from
Jul 5, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
eb6e132
Proxy federation requests
erikjohnston Mar 31, 2023
6a95e7a
Make configurable
erikjohnston Apr 28, 2023
f0270aa
Cache the fed proxy
erikjohnston May 10, 2023
6d98582
Accept a list of federation proxies
erikjohnston May 10, 2023
5889396
Make configurable
erikjohnston May 10, 2023
58fe4da
Comment
erikjohnston May 10, 2023
f00fedd
Remove unused class
erikjohnston May 15, 2023
41c5747
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jun 13, 2023
f219f0e
Add changelog
MadLittleMods Jun 13, 2023
c998d28
Avoid negated condition
MadLittleMods Jun 13, 2023
cc05c97
Fix tests and align to new `matrix-federation://` schema
MadLittleMods Jun 13, 2023
8cfad3d
Fix lints
MadLittleMods Jun 13, 2023
9eec614
WORKER PROXY WIP
erikjohnston May 10, 2023
e9e900f
Align scheme checking
MadLittleMods Jun 14, 2023
dcb4105
Fix lints
MadLittleMods Jun 14, 2023
c6dcd5e
Refactor tests to use `get_clock()`
MadLittleMods Jun 14, 2023
f139898
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jun 14, 2023
8f9f478
Fix tests (make sure `federation_http_client` is defined)
MadLittleMods Jun 14, 2023
e789c64
Fix tests
MadLittleMods Jun 14, 2023
0cead40
Fix lints
MadLittleMods Jun 14, 2023
11bf041
Maybe fix more replication tests
MadLittleMods Jun 14, 2023
d847564
Mark out spots to add docs
MadLittleMods Jun 15, 2023
74988e2
WIP: Very rough worker test
MadLittleMods Jun 16, 2023
6b44e66
Cleaned up test
MadLittleMods Jun 16, 2023
8af2fb8
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jun 20, 2023
1abd3b1
Clean up test
MadLittleMods Jun 20, 2023
477844c
Explain why we care about catching `PotentialDataLoss`
MadLittleMods Jun 20, 2023
dac5532
Add some more context
MadLittleMods Jun 20, 2023
cf208d2
Test error case
MadLittleMods Jun 20, 2023
e665fa8
Flesh out docstrings and comments
MadLittleMods Jun 20, 2023
2ce2025
Update docs
MadLittleMods Jun 20, 2023
632544a
Add some background behind `matrix-federation://`
MadLittleMods Jun 20, 2023
033e18a
Align language
MadLittleMods Jun 20, 2023
b5e916e
Revert back to debug level
MadLittleMods Jun 20, 2023
484680f
Merge branch 'develop' into erikj/fed_proxy
erikjohnston Jun 20, 2023
2032ea6
`master`/`main` is in the `instance_map` so no need to skip checking …
MadLittleMods Jun 21, 2023
926e3e0
Remove extra proxy logging
MadLittleMods Jun 21, 2023
0a2a9cf
Do not copy over hop-by-hop headers
MadLittleMods Jun 21, 2023
c757a38
Add tests for `parse_connection_header_value`
MadLittleMods Jun 21, 2023
be12f21
Add tests to make sure headers are removed
MadLittleMods Jun 21, 2023
735203e
Ignore lint
MadLittleMods Jun 21, 2023
9e3881f
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jun 21, 2023
c1ec014
Fix `arg-type` lint
MadLittleMods Jun 21, 2023
d400b50
Simplify `parse_connection_header_value`
MadLittleMods Jun 21, 2023
e99a5e9
Use safe `json.dumps` for JSON response
MadLittleMods Jun 27, 2023
074fe0c
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jun 28, 2023
d3292d2
Merge branch 'develop' into erikj/fed_proxy
MadLittleMods Jul 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions synapse/config/federation.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,23 @@
# limitations under the License.
from typing import Any, Optional

import attr

from synapse.config._base import Config
from synapse.config._util import validate_config
from synapse.types import JsonDict


@attr.s(frozen=True, auto_attribs=True)
class FederationProxy:
"""A proxy server for outbound federation traffic, for URIs with a
`matrix-federation://` scheme.
"""

host: str
port: int


class FederationConfig(Config):
section = "federation"

Expand Down Expand Up @@ -49,5 +61,12 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
"allow_device_name_lookup_over_federation", False
)

self.federation_proxy = None
federation_proxy = config.get("federation_proxy")
if federation_proxy:
host = federation_proxy["host"]
port = int(federation_proxy["port"])
self.federation_proxy = FederationProxy(host, port)


_METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}}
4 changes: 2 additions & 2 deletions synapse/http/federation/matrix_federation_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def request(
# the host header with the delegated server name.
delegated_server = None
if (
parsed_uri.scheme == b"matrix"
parsed_uri.scheme == b"matrix-federation"
and not _is_ip_literal(parsed_uri.hostname)
and not parsed_uri.port
):
Expand Down Expand Up @@ -379,7 +379,7 @@ async def _resolve_server(self) -> List[Server]:
connect to.
"""

if self._parsed_uri.scheme != b"matrix":
if self._parsed_uri.scheme != b"matrix-federation":
return [Server(host=self._parsed_uri.host, port=self._parsed_uri.port)]

# Note: We don't do well-known lookup as that needs to have happened
Expand Down
29 changes: 24 additions & 5 deletions synapse/http/matrixfederationclient.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
from twisted.internet.task import Cooperator
from twisted.web.client import ResponseFailed
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer, IResponse
from twisted.web.iweb import IAgent, IBodyProducer, IResponse

import synapse.metrics
import synapse.util.retryutils
Expand Down Expand Up @@ -175,7 +175,14 @@ def __attrs_post_init__(self) -> None:

# The object is frozen so we can pre-compute this.
uri = urllib.parse.urlunparse(
(b"matrix", destination_bytes, path_bytes, None, query_bytes, b"")
(
b"matrix-federation",
MadLittleMods marked this conversation as resolved.
Show resolved Hide resolved
destination_bytes,
path_bytes,
None,
query_bytes,
b"",
)
)
object.__setattr__(self, "uri", uri)

Expand Down Expand Up @@ -389,9 +396,21 @@ def __init__(
if hs.config.server.user_agent_suffix:
user_agent = "%s %s" % (user_agent, hs.config.server.user_agent_suffix)

federation_agent = ProxyAgent(
self.reactor, self.reactor, tls_client_options_factory
)
if hs.config.federation.federation_proxy:
federation_agent: IAgent = ProxyAgent(
self.reactor,
self.reactor,
tls_client_options_factory,
federation_proxy=hs.config.federation.federation_proxy,
)
else:
federation_agent = MatrixFederationAgent(
self.reactor,
tls_client_options_factory,
user_agent.encode("ascii"),
hs.config.server.federation_ip_range_whitelist,
hs.config.server.federation_ip_range_blacklist,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we probably want to assert that these options are not in use, if we are using the proxy, since it seems like we do not respect them (right now).

Copy link
Contributor Author

@MadLittleMods MadLittleMods Jul 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think those options will be respected. Workers will delegate outbound federation federation traffic to whatever worker is configured for outbound_federation_restricted_to. And that outbound_federation_restricted_to worker will use the federation_ip_range_allowlist/federation_ip_range_blocklist options.

You have to trust that whatever worker you proxy through but in the Synapse case, I think that's a given (other workers behave as expected).

)

# Use a BlacklistingAgentWrapper to prevent circumventing the IP
# blacklist via IP literals in server names
Expand Down
13 changes: 10 additions & 3 deletions synapse/http/proxyagent.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS, IResponse

from synapse.config.federation import FederationProxy
from synapse.http import redact_uri
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint, ProxyCredentials

Expand Down Expand Up @@ -89,6 +90,7 @@ def __init__(
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
use_proxy: bool = False,
federation_proxy: Optional[FederationProxy] = None,
):
contextFactory = contextFactory or BrowserLikePolicyForHTTPS()

Expand Down Expand Up @@ -126,6 +128,7 @@ def __init__(

self._policy_for_https = contextFactory
self._reactor = reactor
self._federation_proxy = federation_proxy

def request(
self,
Expand Down Expand Up @@ -214,8 +217,12 @@ def request(
parsed_uri.port,
self.https_proxy_creds,
)
elif parsed_uri.scheme == b"matrix":
endpoint = HostnameEndpoint(self.proxy_reactor, "127.0.0.1", 3000)
elif parsed_uri.scheme == b"matrix-federation" and self._federation_proxy:
endpoint = HostnameEndpoint(
self.proxy_reactor,
self._federation_proxy.host,
self._federation_proxy.port,
)
request_path = uri
else:
# not using a proxy
Expand All @@ -236,7 +243,7 @@ def request(
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
elif parsed_uri.scheme == b"matrix":
elif parsed_uri.scheme == b"matrix-federation" and self._federation_proxy:
pass
else:
return defer.fail(
Expand Down