Add support for fetching endpoint list from riseupvpn

ooniapi/services/ooniprobe/src/ooniprobe/models.py

@@ -43,6 +43,9 @@ class OONIProbeVPNProviderEndpoint(Base):
     protocol: Mapped[str] = mapped_column()
     address: Mapped[str] = mapped_column()
     transport: Mapped[str] = mapped_column()
+    # TODO: maybe we want this in the future to store location and other
+    # metadata about an endpoint
+    # metadata: Mapped[Dict[str, str]] = mapped_column(nullable=True)
 
     provider_id = mapped_column(ForeignKey("ooniprobe_vpn_provider.id"))
     provider = relationship("OONIProbeVPNProvider", back_populates="endpoints")

ooniapi/services/ooniprobe/src/ooniprobe/routers/v2.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone, date
+import random
 from typing import Dict, List
 import logging
 
@@ -38,7 +39,6 @@ def update_vpn_provider(db: Session, provider_name: str) -> models.OONIProbeVPNP
     # we are only handling a single provider for the time being (riseup).
     # TODO: manage an inventory of known providers.
     vpn_cert = fetch_openvpn_config()
-    vpn_endpoints = fetch_openvpn_endpoints()
 
     try:
         provider = (
@@ -52,7 +52,13 @@ def update_vpn_provider(db: Session, provider_name: str) -> models.OONIProbeVPNP
         provider.openvpn_cert = vpn_cert["cert"]
         provider.openvpn_key = vpn_cert["key"]
         provider.date_updated = datetime.now(timezone.utc)
-        upsert_endpoints(db, vpn_endpoints, provider)
+
+        try:
+            vpn_endpoints = fetch_openvpn_endpoints()
+            upsert_endpoints(db, vpn_endpoints, provider)
+        except:
+            log.error("Could not fetch endpoints for %s", provider_name)
+
         db.commit()
 
     except sa.orm.exc.NoResultFound:
@@ -65,6 +71,7 @@ def update_vpn_provider(db: Session, provider_name: str) -> models.OONIProbeVPNP
             openvpn_key=vpn_cert["key"],
         )
         db.add(provider)
+        vpn_endpoints = fetch_openvpn_endpoints()
         upsert_endpoints(db, vpn_endpoints, provider)
         db.commit()
 
@@ -106,6 +113,7 @@ def get_vpn_config(
         log.error("Error while fetching credentials for riseup: %s", exc)
         raise HTTPException(status_code=500, detail="could not fetch credentials")
 
+    endpoints = [format_endpoint(provider.provider_name, ep) for ep in provider.endpoints]
     return VPNConfig(
         provider=provider.provider_name,
         protocol="openvpn",
@@ -114,6 +122,7 @@ def get_vpn_config(
             "cert": provider.openvpn_cert,
             "key": provider.openvpn_key,
         },
-        endpoints=[format_endpoint(provider.provider_name, ep) for ep in provider.endpoints],
+        # Pick 4 random endpoints to serve to the client
+        endpoints=random.sample(endpoints, min(len(endpoints), 4)),
         date_updated=provider.date_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
     )

ooniapi/services/ooniprobe/src/ooniprobe/utils.py

@@ -5,6 +5,7 @@
 """
 import base64
 from datetime import datetime, timezone
+import itertools
 import logging
 from typing import Dict, List, Mapping, TypedDict
 
@@ -16,7 +17,7 @@
 
 RISEUP_CA_URL = "https://api.black.riseup.net/ca.crt"
 RISEUP_CERT_URL = "https://api.black.riseup.net/3/cert"
-
+RISEUP_ENDPOINT_URL = "https://api.black.riseup.net/3/config/eip-service.json"
 
 log = logging.getLogger(__name__)
 
@@ -50,24 +51,26 @@ def fetch_openvpn_config() -> OpenVPNConfig:
     return OpenVPNConfig(ca=ca, cert=cert.as_text(), key=key.as_text())
 
 def fetch_openvpn_endpoints() -> List[OpenVPNEndpoint]:
-    # TODO(ain): As a first step, I'm hardcoding a single endpoint. Endpoint discovery
-    # can be done at the same time than credentials renewal, but we probably want
-    # to rotate endpoints more often, design experiments etc, with a different lifecycle
-    # than credentials. A simple implementation can be more or less straightforward,
-    # but we want to dedicate some thought to the data model for the endpoint, since
-    # there might be some extra metadata that we want to expose.
-    return [
-        OpenVPNEndpoint(
-            address="51.15.187.53:1194",
-            transport="udp",
-            protocol="openvpn"
-        ),
-        OpenVPNEndpoint(
-            address="51.15.187.53:1194",
-            transport="tcp",
-            protocol="openvpn"
-        )
-    ]
+    endpoints = []
+
+    r = httpx.get(RISEUP_ENDPOINT_URL)
+    r.raise_for_status()
+    j = r.json()
+    for ep in j["gateways"]:
+        ip = ep["ip_address"]
+        # TODO(art): do we want to store this metadata somewhere?
+        #location = ep["location"]
+        #hostname = ep["host"]
+        for t in ep["capabilities"]["transport"]:
+            if t["type"] != "openvpn":
+                continue
+            for transport, port in itertools.product(t["protocols"], t["ports"]):
+                endpoints.append(OpenVPNEndpoint(
+                    address=f"{ip}:{port}",
+                    protocol="openvpn",
+                    transport=transport
+                ))
+    return endpoints
 
 def format_endpoint(provider_name: str, ep: OONIProbeVPNProviderEndpoint) -> str:
     return f"{ep.protocol}://{provider_name}.corp/?address={ep.address}&transport={ep.transport}"