Switch to split known server lists

See raiden-network/raiden#6442

build/purger/purger.py

@@ -125,7 +125,18 @@ def purge(
  # In case an empty env var is set
  url_known_federation_servers = DEFAULT_MATRIX_KNOWN_SERVERS[Environment.PRODUCTION]
  # fetch remote whiltelist
- remote_whitelist = yaml.safe_load(requests.get(url_known_federation_servers).text)
+ try:
+ remote_whitelist = json.loads(requests.get(url_known_federation_servers).text)[
+ "all_servers"
+ ]
+ except (requests.RequestException, JSONDecodeError, KeyError) as ex:
+ click.secho(
+ f"Error while fetching whitelist: {ex!r}. "
+ f"Ignoring, containers will be restarted.",
+ err=True,
+ )
+ # An empty whitelist will cause the container to be restarted
+ remote_whitelist = []
 
  client = docker.from_env() # pylint: disable=no-member
  for container in client.containers.list():
@@ -145,16 +156,9 @@ def purge(
  continue
 
  click.secho(f"Whitelist changed. Restarting. new_list={remote_whitelist!r}")
- except (
- KeyError,
- IndexError,
- requests.RequestException,
- yaml.scanner.ScannerError,
- ) as ex:
+ except (KeyError, IndexError) as ex:
  click.secho(
- f"An error ocurred while fetching whitelists: {ex!r}\n"
- "Restarting anyway",
- err=True,
+ f"Error fetching container status: {ex!r}. Restarting anyway.", err=True,
  )
  # restart container
  container.restart(timeout=30)

build/room_ensurer/room_ensurer.py

@@ -26,8 +26,6 @@
 
 patch_all() # isort:skip
 
-from raiden.utils.datastructures import merge_dict
-
 import json
 import os
 import sys
@@ -42,7 +40,6 @@
 import gevent
 from eth_utils import encode_hex, to_normalized_address
 from matrix_client.errors import MatrixError
-from raiden_contracts.utils.type_aliases import ChainID
 from structlog import get_logger
 
 from raiden.constants import (
@@ -51,14 +48,16 @@
  PATH_FINDING_BROADCASTING_ROOM,
  Environment,
  Networks,
+ ServerListType,
 )
 from raiden.log_config import configure_logging
 from raiden.network.transport.matrix import make_room_alias
 from raiden.network.transport.matrix.client import GMatrixHttpApi
 from raiden.settings import DEFAULT_MATRIX_KNOWN_SERVERS
 from raiden.tests.utils.factories import make_signer
 from raiden.utils.cli import get_matrix_servers
-
+from raiden.utils.datastructures import merge_dict
+from raiden_contracts.utils.type_aliases import ChainID
 
 ENV_KEY_KNOWN_SERVERS = "URL_KNOWN_FEDERATION_SERVERS"
 
@@ -104,7 +103,9 @@ def __init__(
 
  self._known_servers: Dict[str, str] = {
  urlparse(server_url).netloc: server_url
- for server_url in get_matrix_servers(known_servers_url)
+ for server_url in get_matrix_servers(
+ known_servers_url, server_list_type=ServerListType.ALL_SERVERS
+ )
  }
  if not self._known_servers:
  raise RuntimeError(f"No known servers found from list at {known_servers_url}.")
@@ -214,7 +215,7 @@ def _ensure_room_for_network(self, network: Networks, alias_fragment: str) -> No
  self._ensure_admin_power_levels(room_infos[self._own_server_name])
 
  def _join_and_alias_room(
-  self, first_server_room_alias: str, own_server_room_alias: str
+ self, first_server_room_alias: str, own_server_room_alias: str
  ) -> None:
  response = self._own_api.join_room(first_server_room_alias)
  own_room_id = response.get("room_id")
@@ -291,12 +292,16 @@ def _ensure_admin_power_levels(self, room_info: RoomInfo) -> None:
  return
 
  if own_user not in current_power_levels["users"]:
- log.warning(f"{own_user} has not been granted administrative power levels yet. Doing nothing.")
+ log.warning(
+ f"{own_user} has not been granted administrative power levels yet. Doing nothing."
+ )
  return
 
  # the supposed power level dict could be just a subset of the current
  # because providers who left cannot be removed from other admins
- if set(supposed_power_levels["users"].keys()).issubset(set(current_power_levels["users"].keys())):
+ if set(supposed_power_levels["users"].keys()).issubset(
+ set(current_power_levels["users"].keys())
+ ):
  log.debug(f"Power levels are up to date. Doing nothing.")
  return
 

build/synapse/render_config_template.py

@@ -2,16 +2,23 @@
 import os
 import random
 import string
+from json.decoder import JSONDecodeError
 from pathlib import Path
 from typing import Optional
 from urllib.error import URLError
 from urllib.request import urlopen
 
-PATH_CONFIG = Path("/config/synapse.yaml")
-PATH_CONFIG_TEMPLATE = Path("/config/synapse.template.yaml")
+import click
+from docker import Client
+
+PATH_CONFIG_SYNAPSE = Path("/config/synapse.yaml")
+PATH_CONFIG_TEMPLATE_SYNAPSE = Path("/config/synapse.template.yaml")
+PATH_CONFIG_WORKER_BASE = Path("/config/workers/")
+PATH_CONFIG_TEMPLATE_WORKER = Path("/config/worker.template.yaml")
+
 PATH_MACAROON_KEY = Path("/data/keys/macaroon.key")
 PATH_ADMIN_USER_CREDENTIALS = Path("/config/admin_user_cred.json")
-PATH_KNOWN_FEDERATION_SERVERS = Path("/data/known_federation_servers.yaml")
+PATH_KNOWN_FEDERATION_SERVERS = Path("/data/known_federation_servers.json")
 PATH_WELL_KNOWN_FILE = Path("/data_well_known/server")
 
 # This file gets created during docker build from the given Raiden version
@@ -36,7 +43,13 @@ def get_known_federation_servers(url_known_federation_servers: Optional[str]) ->
  try:
  resp = urlopen(url_known_federation_servers)
  if 200 <= resp.code < 300:
- PATH_KNOWN_FEDERATION_SERVERS.write_text(resp.read().decode())
+ try:
+ known_servers = json.loads(resp.read().decode())
+ PATH_KNOWN_FEDERATION_SERVERS.write_text(
+ "".join(f"- {server}\n" for server in known_servers["all_servers"])
+ )
+ except (JSONDecodeError, KeyError):
+ print("Error loading known servers list:", resp.code, resp.read().decode())
  else:
  print("Error fetching known servers list:", resp.code, resp.read().decode())
  except URLError as ex:
@@ -47,13 +60,13 @@ def get_known_federation_servers(url_known_federation_servers: Optional[str]) ->
 
 
 def render_synapse_config(server_name: str, url_known_federation_servers: Optional[str]) -> None:
- template_content = PATH_CONFIG_TEMPLATE.read_text()
+ template_content = PATH_CONFIG_TEMPLATE_SYNAPSE.read_text()
  rendered_config = string.Template(template_content).substitute(
  MACAROON_KEY=get_macaroon_key(),
  SERVER_NAME=server_name,
  KNOWN_SERVERS=get_known_federation_servers(url_known_federation_servers),
  )
- PATH_CONFIG.write_text(rendered_config)
+ PATH_CONFIG_SYNAPSE.write_text(rendered_config)
 
 
 def render_well_known_file(server_name: str) -> None:

known_servers/known_servers-development-v1.2.0.json

@@ -0,0 +1,14 @@
+{
+ "active_servers": [
+ "transport.transport01.raiden.network",
+ "transport.transport02.raiden.network",
+ "transport.transport03.raiden.network",
+ "transport.transport04.raiden.network"
+ ],
+ "all_servers": [
+ "transport.transport01.raiden.network",
+ "transport.transport02.raiden.network",
+ "transport.transport03.raiden.network",
+ "transport.transport04.raiden.network"
+ ]
+}

known_servers/known_servers-production-v1.2.0.json

@@ -0,0 +1,13 @@
+{
+ "active_servers": [
+ "transport.mainnet.raiden.anyblock.tools",
+ "transport.raiden.dappnode.io",
+ "transport.raiden.badgateway.tech"
+ ],
+ "all_servers": [
+ "transport.mainnet.raiden.anyblock.tools",
+ "transport.raiden.dappnode.io",
+ "transport.raiden.badgateway.tech",
+ "transport.raiden.overdoze.se"
+ ]
+}