Reticulum pre-TURN Hubs Cloud 4/21 Update

README.md

@@ -4,7 +4,7 @@ A hybrid game networking and web API server, focused on Social Mixed Reality.
 
 ## Development
 
-### Install Prerequisite Packages:
+### 1. Install Prerequisite Packages:
 #### PostgreSQL (recommended version 11.x):
 Linux: Use your package manager
 
@@ -16,7 +16,7 @@ Windows WSL: https://github.com/michaeltreat/Windows-Subsystem-For-Linux-Setup-G
 https://elixir-lang.org/install.html
 https://hexdocs.pm/phoenix/installation.html
 
-### Setup Reticulum:
+### 2. Setup Reticulum:
 Run the following commands at the root of the reticulum directory:
 1. `mix deps.get`
 2. `mix ecto.create`
@@ -25,17 +25,59 @@ Run the following commands at the root of the reticulum directory:
 3. from the `assets` directory, `npm install`
 4. From the project directory `mkdir -p storage/dev`
 
-### Start Reticulum
+### 3. Start Reticulum
 Run `scripts/run.sh` if you have the hubs secret repo cloned. Otherwise `iex -S mix phx.server`
 
 ## Run Hubs Against a Local Reticulum Instance
-1. Clone and start hubs by running `./scripts/run_local_reticulum.sh` in the root of the hubs project
-2. Go to https://hubs.local:4000?somerandomvar (note the random query string)
-3. To sign in click the sign in link and submit your email.
-4. Go to the reticulum terminal session and find a url that looks like https://hubs.local:4000/?auth_origin=hubs&auth_payload=XXXXX&auth_token=XXXX
-5. Navigate to that url in your browser to finish signing in.
 
-Adter you've started Reticulum for the first time you'll likely want to create an admin user. Assuming you want to make the first account the admin, this can be done in the iex console using the following code:
+### 0. Dependencies
+
+[Install NodeJS](https://nodejs.org) if you haven't already. We recommend version 12 or above.
+
+### 1. Setup the `hubs.local` hostname
+
+When running the full stack for Hubs (which includes Reticulum) locally it is necessary to add a `hosts` entry pointing `hubs.local` to your local server's IP.
+This will allow the CSP checks to pass that are served up by Reticulum so you can test the whole app. Note that you must also load hubs.local over https.
+
+Example:
+```
+hubs.local 127.0.0.1
+```
+
+### 2. Setting up the Hubs Repository
+
+Clone the Hubs repository and install the npm dependencies.
+
+```bash
+git clone https://github.com/mozilla/hubs.git
+cd hubs
+npm ci
+```
+
+### 3. Start the Hubs Webpack Dev Server
+
+Because we are running Hubs against the local Reticulum client you'll need to use the `npm run local` command in the root of the `hubs` folder. This will start the development server on port 8080, but configure it to be accessed through Reticulum on port 4000.
+
+### 4. Navigate To The Client Page
+
+Once both the Hubs Webpack Dev Server and Reticulum server are both running you can navigate to the client by opening up:
+
+https://hubs.local:4000?skipadmin
+
+> The `skipadmin` is a temporary measure to bypass being redirected to the admin panel. Once you have logged in you will no longer need this.
+
+### 5. Logging In
+
+To log into Hubs we use magic links that are sent to your email. When you are running Reticulum locally we do not send those emails. Instead, you'll find the contents of that email in the Reticulum console output.
+
+With the Hubs landing page open click the Sign In button at the top of the page. Enter an email address and click send.
+
+Go to the reticulum terminal session and find a url that looks like https://hubs.local:4000/?auth_origin=hubs&auth_payload=XXXXX&auth_token=XXXX
+
+Navigate to that url in your browser to finish signing in.
+
+### 6. Creating an Admin User
+After you've started Reticulum for the first time you'll likely want to create an admin user. Assuming you want to make the first account the admin, this can be done in the iex console using the following code:
 
 ```
 Ret.Account |> Ret.Repo.all() |> Enum.at(0) |> Ecto.Changeset.change(is_admin: true) |> Ret.Repo.update!()

config/config.exs

@@ -6,7 +6,7 @@
 use Mix.Config
 
 # General application configuration
-config :ret, ecto_repos: [Ret.Repo]
+config :ret, ecto_repos: [Ret.Repo, Ret.SessionLockRepo]
 
 config :phoenix, :format_encoders, "json-api": Jason
 config :phoenix, :json_library, Jason
@@ -45,6 +45,14 @@ config :ret, Ret.Repo,
   ownership_timeout: 60_000,
   timeout: 60_000
 
+config :ret, Ret.SessionLockRepo,
+  migration_source: "schema_migrations",
+  migration_default_prefix: "ret0",
+  after_connect: {Ret.SessionLockRepo, :set_search_path, ["public, ret0"]},
+  # Downloads from Sketchfab to file cache hold connections open
+  ownership_timeout: 60_000,
+  timeout: 60_000
+
 config :peerage, log_results: false
 
 config :statix, prefix: "ret"

config/dev.exs

@@ -93,6 +93,14 @@ config :ret, Ret.Repo,
   template: "template0",
   pool_size: 10
 
+config :ret, Ret.SessionLockRepo,
+  username: "postgres",
+  password: "postgres",
+  database: "ret_dev",
+  hostname: if(env_db_host == "", do: "localhost", else: env_db_host),
+  template: "template0",
+  pool_size: 10
+
 config :ret, RetWeb.Plugs.HeaderAuthorization,
   header_name: "x-ret-admin-access-key",
   header_value: "admin-only"

config/prod.exs

@@ -44,6 +44,14 @@ config :ret, Ret.Repo,
   template: "template0",
   pool_size: 10
 
+config :ret, Ret.SessionLockRepo,
+  username: "postgres",
+  password: "postgres",
+  database: "ret_production",
+  hostname: "localhost",
+  template: "template0",
+  pool_size: 10
+
 # ## SSL Support
 #
 # To get SSL working, you will need to add the `https` key

config/test.exs

@@ -12,10 +12,22 @@ config :ret, RetWeb.Endpoint,
 # Print only warnings and errors during test
 config :logger, level: :warn
 
+db_credentials = "admin"
+
 config :ret, Ret.Repo,
   adapter: Ecto.Adapters.Postgres,
-  username: "admin",
-  password: "admin",
+  username: db_credentials,
+  password: db_credentials,
+  database: "ret_test",
+  hostname: "localhost",
+  template: "template0",
+  pool_size: 10,
+  pool: Ecto.Adapters.SQL.Sandbox
+
+config :ret, Ret.SessionLockRepo,
+  adapter: Ecto.Adapters.Postgres,
+  username: db_credentials,
+  password: db_credentials,
   database: "ret_test",
   hostname: "localhost",
   template: "template0",
@@ -25,8 +37,8 @@ config :ret, Ret.Repo,
 config :ret, Ret.Locking,
   lock_timeout_ms: 1000 * 60 * 15,
   session_lock_db: [
-    username: "admin",
-    password: "admin",
+    username: db_credentials,
+    password: db_credentials,
     database: "ret_test",
     hostname: "localhost"
   ]

habitat/config/config.toml

@@ -112,6 +112,28 @@ port = {{ cfg.db.port }}
 ssl = {{ cfg.db.ssl }}
 {{/if}}
 
+[ret."Elixir.Ret.SessionLockRepo"]
+username = "{{ cfg.session_lock_db.username }}"
+password = "{{ cfg.session_lock_db.password }}"
+database = "{{ cfg.session_lock_db.database }}"
+hostname = "{{ cfg.session_lock_db.hostname }}"
+template = "{{ cfg.db.template }}"
+{{#if cfg.db.queue_interval }}
+queue_interval = {{ cfg.db.queue_interval }}
+{{/if}}
+{{#if cfg.db.queue_target }}
+queue_target = {{ cfg.db.queue_target }}
+{{/if}}
+{{#if cfg.db.timeout }}
+timeout = {{ cfg.db.timeout }}
+{{/if}}
+{{#if cfg.session_lock_db.port }}
+port = {{ cfg.session_lock_db.port }}
+{{/if}}
+{{#if cfg.session_lock_db.ssl }}
+ssl = {{ cfg.session_lock_db.ssl }}
+{{/if}}
+
 [ret."Elixir.Ret.Locking".session_lock_db]
 username = "{{ cfg.session_lock_db.username }}"
 password = "{{ cfg.session_lock_db.password }}"

lib/ret/application.ex

@@ -11,39 +11,54 @@ defmodule Ret.Application do
     Application.load(:ret)
     EctoBootMigration.start_dependencies()
 
-    repos_pids = EctoBootMigration.start_repos([Ret.Repo])
+    repos_pids =
+      Ret.Locking.exec_if_session_lockable("ret_migration", fn ->
+        repos_pids = EctoBootMigration.start_repos([Ret.SessionLockRepo])
 
-    Ret.Locking.exec_if_session_lockable("ret_migration", fn ->
-      db_name = Application.get_env(:ret, Ret.Repo)[:database]
+        # Note the main Repo database is used here, since the session locking database
+        # name may be a proxy database in pgbouncer which doesn't actually exist.
+        db_name = Application.get_env(:ret, Ret.Repo)[:database]
 
-      # Can't check mix_env here, so check db name
-      if db_name !== "ret_test" do
-        coturn_enabled = Ret.Coturn.enabled?()
+        # Can't check mix_env here, so check db name
+        if db_name !== "ret_test" do
+          coturn_enabled = Ret.Coturn.enabled?()
 
-        Ecto.Adapters.SQL.query!(Ret.Repo, "CREATE SCHEMA IF NOT EXISTS ret0")
+          Ecto.Adapters.SQL.query!(Ret.SessionLockRepo, "CREATE SCHEMA IF NOT EXISTS ret0")
 
-        if coturn_enabled do
-          Ecto.Adapters.SQL.query!(Ret.Repo, "CREATE SCHEMA IF NOT EXISTS coturn")
-        end
-
-        schemas =
           if coturn_enabled do
-            ["ret0", "coturn"]
-          else
-            ["ret0"]
+            Ecto.Adapters.SQL.query!(Ret.SessionLockRepo, "CREATE SCHEMA IF NOT EXISTS coturn")
+          end
+
+          Ecto.Adapters.SQL.query!(
+            Ret.SessionLockRepo,
+            "ALTER DATABASE #{db_name} SET search_path TO ret0"
+          )
+
+          priv_path = Path.join(["#{:code.priv_dir(:ret)}", "repo", "migrations"])
+
+          # Disallow stop of the application via SIGTERM until migrations are finished.
+          #
+          # If application is killed mid-migration, then it's possible for schema migrations
+          # table to not accurately reflect the migrations which have ran.
+          Ret.DelayStopSignalHandler.delay_stop()
+
+          try do
+            Ecto.Migrator.run(Ret.SessionLockRepo, priv_path, :up, all: true, prefix: "ret0")
+          after
+            Ret.DelayStopSignalHandler.allow_stop()
           end
 
-        Ecto.Adapters.SQL.query!(
-          Ret.Repo,
-          "ALTER DATABASE #{db_name} SET search_path TO #{schemas |> Enum.join(",")}"
-        )
+          repos_pids
+        end
+      end)
 
-        priv_path = Path.join(["#{:code.priv_dir(:ret)}", "repo", "migrations"])
-        Ecto.Migrator.run(Ret.Repo, priv_path, :up, all: true, prefix: "ret0")
-      end
-    end)
+    if repos_pids do
+      # Ensure there are some TURN secrets in the database, so that if system is idle
+      # the cron isn't indefinitely skipped and nobody can join rooms.
+      Ret.Coturn.rotate_secrets(true, Ret.SessionLockRepo)
 
-    EctoBootMigration.stop_repos(repos_pids)
+      EctoBootMigration.stop_repos(repos_pids)
+    end
 
     :ok = Ret.Statix.connect()
 

lib/ret/coturn.ex

@@ -1,17 +1,17 @@
 defmodule Ret.Coturn do
   # Adds a new secret, and removes secrets older than 15 minutes since a new one is generated every five.
   # Note this is safe to run on a multi-node cluster since coturn respects all secrets in the db.
-  def rotate_secrets do
+  def rotate_secrets(force \\ false, repo \\ Ret.Repo) do
     # Don't perform database cron if turn is disabled or nobody is connected, to prevent un-pausing db.
-    if enabled?() && RetWeb.Presence.has_present_members?() do
+    if enabled?() && (force || RetWeb.Presence.has_present_members?()) do
       Ecto.Adapters.SQL.query!(
-        Ret.Repo,
+        repo,
         "INSERT INTO coturn.turn_secret (realm, value, inserted_at, updated_at) values ($1, $2, now(), now())",
         [realm(), SecureRandom.hex()]
       )
 
       Ecto.Adapters.SQL.query!(
-        Ret.Repo,
+        repo,
         "DELETE FROM coturn.turn_secret WHERE inserted_at < now() - interval '15 minutes'"
       )
     end

lib/ret/delay_stop_signal_handler.ex

@@ -0,0 +1,39 @@
+# This gen_event handler allows a temporary delay of handling
+# SIGTERM to shut down the application.
+#
+# Calling delay_stop will cease handling SIGTERM, then calling
+# allow_stop will resume it. If SIGTERM was seen in the interim,
+# the application will then stop.
+defmodule Ret.DelayStopSignalHandler do
+  @moduledoc false
+
+  def init(_), do: {:ok, %{saw_sigterm: false}}
+
+  def delay_stop do
+    :ok =
+      :gen_event.swap_sup_handler(
+        :erl_signal_server,
+        {:erl_signal_handler, []},
+        {Ret.DelayStopSignalHandler, []}
+      )
+  end
+
+  def allow_stop do
+    :gen_event.call(:erl_signal_server, Ret.DelayStopSignalHandler, {:allow_stop, self()})
+
+    :ok =
+      :gen_event.swap_sup_handler(
+        :erl_signal_server,
+        {Ret.DelayStopSignalHandler, []},
+        {:erl_signal_handler, []}
+      )
+  end
+
+  def handle_call({:allow_stop, _pid}, %{saw_sigterm: true} = state) do
+    :init.stop()
+    {:ok, :ok, state}
+  end
+
+  def handle_call({:allow_stop, _pid}, state), do: {:ok, :ok, state}
+  def handle_event(:sigterm, state), do: {:ok, state |> Map.put(:saw_sigterm, true)}
+end

lib/ret/hub.ex

@@ -439,7 +439,7 @@ defmodule Ret.Hub do
       transports =
         (Application.get_env(:ret, Ret.Coturn)[:public_tls_ports] || "5349")
         |> String.split(",")
-        |> Enum.map(&%{transport: :tls, port: &1 |> Integer.parse() |> elem(0)})
+        |> Enum.map(&%{port: &1 |> Integer.parse() |> elem(0)})
 
       %{enabled: true, username: username, credential: credential, transports: transports}
     else

lib/ret/session_lock_repo.ex

@@ -0,0 +1,14 @@
+# This repo is intended to be the same as Ret.Repo, except statements/transactions
+# execute in a session context (vs a transaction context) when running in an environment
+# with pgbouncer.
+defmodule Ret.SessionLockRepo do
+  use Ecto.Repo, otp_app: :ret, adapter: Ecto.Adapters.Postgres
+
+  def init(_, opts) do
+    {:ok, opts}
+  end
+
+  def set_search_path(conn, path) do
+    {:ok, _result} = Postgrex.query(conn, "set search_path=#{path}", [])
+  end
+end

mix.exs

@@ -38,8 +38,9 @@ defmodule Ret.Mixfile do
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
       {:plug, "~> 1.7"},
-      {:ecto, "~> 3.1"},
-      {:ecto_sql, "~> 3.1"},
+      # Avoid 3.4.0 for now bc https://github.com/elixir-ecto/ecto/issues/3246
+      {:ecto, "~> 3.3.0"},
+      {:ecto_sql, "~> 3.3.0"},
       {:postgrex, ">= 0.0.0"},
       {:phoenix_html, "~> 2.13"},
       {:phoenix_live_reload, "~> 1.2", only: :dev},
@@ -57,7 +58,7 @@ defmodule Ret.Mixfile do
       {:credo, "~> 1.1", only: [:dev, :test], runtime: false},
       {:plug_attack, "~> 0.4"},
       {:ecto_enum, "~> 1.3"},
-      {:the_end, "~> 1.1.0"},
+      {:the_end, git: "https://github.com/mozillareality/the_end.git", branch: "bug/phoenix-14"},
       {:cachex, "~> 3.2"},
       {:retry, "~> 0.13"},
       {:open_graph, "~> 0.0.3"},