From bdcbe4d0dbbf7fa600c98fe6a8cbae4b4dfea3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Thu, 1 Aug 2024 17:37:56 +0200 Subject: [PATCH 1/7] Allow CORS --- broadcaster/lib/broadcaster_web/endpoint.ex | 2 ++ broadcaster/mix.exs | 1 + broadcaster/mix.lock | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/broadcaster/lib/broadcaster_web/endpoint.ex b/broadcaster/lib/broadcaster_web/endpoint.ex index f8ddb4f..e2e3a85 100644 --- a/broadcaster/lib/broadcaster_web/endpoint.ex +++ b/broadcaster/lib/broadcaster_web/endpoint.ex @@ -19,6 +19,8 @@ defmodule BroadcasterWeb.Endpoint do websocket: [connect_info: [session: @session_options]], longpoll: [connect_info: [session: @session_options]] + plug Corsica, origins: "*" + # Serve at "/" the static files from "priv/static" directory. # # You should set gzip to true if you are running phx.digest diff --git a/broadcaster/mix.exs b/broadcaster/mix.exs index a00ac81..b548e25 100644 --- a/broadcaster/mix.exs +++ b/broadcaster/mix.exs @@ -58,6 +58,7 @@ defmodule Broadcaster.MixProject do {:jason, "~> 1.2"}, {:dns_cluster, "~> 0.1.1"}, {:bandit, "~> 1.2"}, + {:corsica, "~> 2.1.3"}, {:ex_webrtc, github: "elixir-webrtc/ex_webrtc", override: true}, {:ex_webrtc_dashboard, github: "elixir-webrtc/ex_webrtc_dashboard"}, diff --git a/broadcaster/mix.lock b/broadcaster/mix.lock index e1c38ed..5331141 100644 --- a/broadcaster/mix.lock +++ b/broadcaster/mix.lock @@ -5,6 +5,7 @@ "bundlex": {:hex, :bundlex, "1.5.3", "35d01e5bc0679510dd9a327936ffb518f63f47175c26a35e708cc29eaec0890b", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:qex, "~> 0.5", [hex: :qex, repo: "hexpm", optional: false]}, {:req, ">= 0.4.0", [hex: :req, repo: "hexpm", optional: false]}, {:zarex, "~> 1.0", [hex: :zarex, repo: "hexpm", optional: false]}], "hexpm", "debd0eac151b404f6216fc60222761dff049bf26f7d24d066c365317650cd118"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "corsica": {:hex, :corsica, "2.1.3", "dccd094ffce38178acead9ae743180cdaffa388f35f0461ba1e8151d32e190e6", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "616c08f61a345780c2cf662ff226816f04d8868e12054e68963e95285b5be8bc"}, "crc": {:hex, :crc, "0.10.5", "ee12a7c056ac498ef2ea985ecdc9fa53c1bfb4e53a484d9f17ff94803707dfd8", [:mix, :rebar3], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3e673b6495a9525c5c641585af1accba59a1eb33de697bedf341e247012c2c7f"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, @@ -21,7 +22,7 @@ "ex_sdp": {:hex, :ex_sdp, "0.17.0", "4c50e7814f01f149c0ccf258fba8428f8567dffecf1c416ec3f6aaaac607a161", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "c7fe0625902be2a835b5fe6834a189f7db7639d2625c8e9d8b3564e6d704145f"}, "ex_stun": {:hex, :ex_stun, "0.2.0", "feb1fc7db0356406655b2a617805e6c712b93308c8ea2bf0ba1197b1f0866deb", [:mix], [], "hexpm", "1e01ba8290082ccbf37acaa5190d1f69b51edd6de2026a8d6d51368b29d115d0"}, "ex_turn": {:hex, :ex_turn, "0.1.0", "177405aadf3d754567d0d37cf881a83f9cacf8f45314d188633b04c4a9e7c1ec", [:mix], [{:ex_stun, "~> 0.2.0", [hex: :ex_stun, repo: "hexpm", optional: false]}], "hexpm", "d677737fb7d45274d5dac19fe3c26b9038b6effbc0a6b3e7417bccc76b6d1cd3"}, - "ex_webrtc": {:git, "https://github.com/elixir-webrtc/ex_webrtc.git", "ef355c055574f498afd448783d7fd510da7b4d20", []}, + "ex_webrtc": {:git, "https://github.com/elixir-webrtc/ex_webrtc.git", "af1fec40b83c8c728073431c6a2b818ecaf974d6", []}, "ex_webrtc_dashboard": {:git, "https://github.com/elixir-webrtc/ex_webrtc_dashboard.git", "40b67a6399dab5eebaa0434edad74fbc747187bd", []}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, From 7bb1e1ed51d22c48028c6a7bea5aae367767517f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Thu, 1 Aug 2024 17:38:17 +0200 Subject: [PATCH 2/7] Rewrite the headless client to not use the admin panel --- broadcaster/.gitignore | 5 ++ broadcaster/headless_client.js | 90 ++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/broadcaster/.gitignore b/broadcaster/.gitignore index c235c0a..c8177f9 100644 --- a/broadcaster/.gitignore +++ b/broadcaster/.gitignore @@ -36,3 +36,8 @@ npm-debug.log /assets/node_modules/ /_dialyzer/ + +# artifacts of the headless_client.js +/node_modules +package.json +package-lock.json diff --git a/broadcaster/headless_client.js b/broadcaster/headless_client.js index 4abacbe..0fbb91d 100644 --- a/broadcaster/headless_client.js +++ b/broadcaster/headless_client.js @@ -2,7 +2,10 @@ const puppeteer = require('puppeteer'); -(async () => { +const url = (process.env.URL === undefined) ? 'http://localhost:4000' : process.env.URL; +const token = (process.env.TOKEN === undefined) ? 'example' : process.env.TOKEN; + +async function start() { const browser = await puppeteer.launch({ // headless: false, args: [ @@ -11,36 +14,59 @@ const puppeteer = require('puppeteer'); '--use-fake-device-for-media-stream' ] }); - - try { - const username = (process.env.USERNAME === undefined) ? 'admin' : process.env.USERNAME; - const password = (process.env.PASSWORD === undefined) ? 'admin' : process.env.PASSWORD; - const url = (process.env.URL === undefined) ? 'http://localhost:4000' : process.env.URL; - const token = (process.env.TOKEN === undefined) ? 'example' : process.env.TOKEN; - - const page = await browser.newPage(); - await page.setViewport({width: 1280, height: 720}); - await page.authenticate({username: username, password: password}); - await page.goto(`${url}/admin/player`); - - // When button is available and initialized, - // we can safely start streaming. - await page.waitForSelector('button'); - await page.waitForFunction(() => { - const button = document.getElementById('button'); - console.log(button); - return button.onclick !== null; + + const page = await browser.newPage(); + // we need a page with secure context in order to access userMedia + await page.goto(`${url}/notfound`); + await page.evaluate(async (url, token) => { + const localStream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { ideal: 1280 }, + height: { ideal: 720 }, + frameRate: { ideal: 24 }, + }, + audio: true, + }); + + const pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); + pc.onconnectionstatechange = async (_) => { + if (pc.connectionState === "failed") { + await browser.close(); + start(); + } + }; + + pc.addTrack(localStream.getAudioTracks()[0], localStream); + pc.addTransceiver(localStream.getVideoTracks()[0], { + streams: [localStream], + sendEncodings: [ + { rid: 'h', maxBitrate: 1500 * 1024 }, + { rid: 'm', scaleResolutionDownBy: 2, maxBitrate: 600 * 1024 }, + { rid: 'l', scaleResolutionDownBy: 4, maxBitrate: 300 * 1024 }, + ], }); - - await page.evaluate((url, token) => { - document.getElementById('serverUrl').value = `${url}/api/whip`; - document.getElementById('serverToken').value = token; - }, url, token); - - await page.evaluate(() => { - document.getElementById('button').click(); + + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + + const response = await fetch(`${url}/api/whip`, { + method: 'POST', + cache: 'no-cache', + headers: { + Accept: 'application/sdp', + 'Content-Type': 'application/sdp', + Authorization: `Bearer ${token}`, + }, + body: offer.sdp, }); - } catch { - await browser.close(); - } -})(); \ No newline at end of file + + if (response.status !== 201) { + throw Error("Unable to connect to the server"); + } + + const sdp = await response.text(); + await pc.setRemoteDescription({ type: 'answer', sdp: sdp }); + }, url, token); +} + +start(); From 13cf19bd098ef41ee471ac1d3b9c8583f1851707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Thu, 1 Aug 2024 17:40:25 +0200 Subject: [PATCH 3/7] Turn on more logs --- broadcaster/config/config.exs | 2 +- broadcaster/config/prod.exs | 2 +- broadcaster/headless_client.js | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/broadcaster/config/config.exs b/broadcaster/config/config.exs index 216ffe1..3ec89e9 100644 --- a/broadcaster/config/config.exs +++ b/broadcaster/config/config.exs @@ -47,7 +47,7 @@ config :tailwind, config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id], - level: :info + level: :debug # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason diff --git a/broadcaster/config/prod.exs b/broadcaster/config/prod.exs index 49d7c24..62a0773 100644 --- a/broadcaster/config/prod.exs +++ b/broadcaster/config/prod.exs @@ -9,7 +9,7 @@ config :broadcaster, BroadcasterWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json" # Do not print debug messages in production -config :logger, level: :info +config :logger, level: :debug # Runtime production configuration, including reading # of environment variables, is done on config/runtime.exs. diff --git a/broadcaster/headless_client.js b/broadcaster/headless_client.js index 0fbb91d..d9bf4a2 100644 --- a/broadcaster/headless_client.js +++ b/broadcaster/headless_client.js @@ -8,6 +8,7 @@ const token = (process.env.TOKEN === undefined) ? 'example' : process.env.TOKEN; async function start() { const browser = await puppeteer.launch({ // headless: false, + dumpio: true, args: [ '--no-sandbox', '--use-fake-ui-for-media-stream', From bc4eef0d4410e814b69aa2dd89c22fb301de9bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Thu, 1 Aug 2024 18:13:54 +0200 Subject: [PATCH 4/7] More logs --- broadcaster/headless_client.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/broadcaster/headless_client.js b/broadcaster/headless_client.js index d9bf4a2..65b8483 100644 --- a/broadcaster/headless_client.js +++ b/broadcaster/headless_client.js @@ -16,7 +16,10 @@ async function start() { ] }); + console.log("Initialising the client..."); + const page = await browser.newPage(); + page.on('console', msg => console.log('PAGE LOG:', msg.text())); // we need a page with secure context in order to access userMedia await page.goto(`${url}/notfound`); await page.evaluate(async (url, token) => { @@ -31,12 +34,15 @@ async function start() { const pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); pc.onconnectionstatechange = async (_) => { + console.log("Connection state changed:", pc.connectionState); if (pc.connectionState === "failed") { await browser.close(); start(); } }; + pc.onicegatheringstatechange = (_) => console.log("ICE gathering state changed:", pc.iceGatheringState); + pc.addTrack(localStream.getAudioTracks()[0], localStream); pc.addTransceiver(localStream.getVideoTracks()[0], { streams: [localStream], From d70af5a0427a35215fec34fd6ea3744f8420479c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Fri, 2 Aug 2024 10:44:55 +0200 Subject: [PATCH 5/7] On connection failed, create new PeerConnection on the same page, instad of new page --- broadcaster/headless_client.js | 137 ++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 64 deletions(-) diff --git a/broadcaster/headless_client.js b/broadcaster/headless_client.js index 65b8483..3709d3f 100644 --- a/broadcaster/headless_client.js +++ b/broadcaster/headless_client.js @@ -1,79 +1,88 @@ -'use strict'; +"use strict"; -const puppeteer = require('puppeteer'); +const puppeteer = require("puppeteer"); -const url = (process.env.URL === undefined) ? 'http://localhost:4000' : process.env.URL; -const token = (process.env.TOKEN === undefined) ? 'example' : process.env.TOKEN; +const url = + process.env.URL === undefined ? "http://localhost:4000" : process.env.URL; +const token = process.env.TOKEN === undefined ? "example" : process.env.TOKEN; -async function start() { - const browser = await puppeteer.launch({ - // headless: false, - dumpio: true, - args: [ - '--no-sandbox', - '--use-fake-ui-for-media-stream', - '--use-fake-device-for-media-stream' - ] - }); +async function stream(url, token) { + console.log("Starting new stream..."); - console.log("Initialising the client..."); + const localStream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { ideal: 1280 }, + height: { ideal: 720 }, + frameRate: { ideal: 24 }, + }, + audio: true, + }); - const page = await browser.newPage(); - page.on('console', msg => console.log('PAGE LOG:', msg.text())); - // we need a page with secure context in order to access userMedia - await page.goto(`${url}/notfound`); - await page.evaluate(async (url, token) => { - const localStream = await navigator.mediaDevices.getUserMedia({ - video: { - width: { ideal: 1280 }, - height: { ideal: 720 }, - frameRate: { ideal: 24 }, - }, - audio: true, - }); + const pc = new RTCPeerConnection({ + iceServers: [{ urls: "stun:stun.l.google.com:19302" }], + }); + pc.onconnectionstatechange = async (_) => { + console.log("Connection state changed:", pc.connectionState); + if (pc.connectionState === "failed") { + stream(url, token); + } + }; - const pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); - pc.onconnectionstatechange = async (_) => { - console.log("Connection state changed:", pc.connectionState); - if (pc.connectionState === "failed") { - await browser.close(); - start(); - } - }; + pc.addTrack(localStream.getAudioTracks()[0], localStream); + pc.addTransceiver(localStream.getVideoTracks()[0], { + streams: [localStream], + sendEncodings: [ + { rid: "h", maxBitrate: 1500 * 1024 }, + { rid: "m", scaleResolutionDownBy: 2, maxBitrate: 600 * 1024 }, + { rid: "l", scaleResolutionDownBy: 4, maxBitrate: 300 * 1024 }, + ], + }); - pc.onicegatheringstatechange = (_) => console.log("ICE gathering state changed:", pc.iceGatheringState); + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); - pc.addTrack(localStream.getAudioTracks()[0], localStream); - pc.addTransceiver(localStream.getVideoTracks()[0], { - streams: [localStream], - sendEncodings: [ - { rid: 'h', maxBitrate: 1500 * 1024 }, - { rid: 'm', scaleResolutionDownBy: 2, maxBitrate: 600 * 1024 }, - { rid: 'l', scaleResolutionDownBy: 4, maxBitrate: 300 * 1024 }, - ], - }); + const response = await fetch(`${url}/api/whip`, { + method: "POST", + cache: "no-cache", + headers: { + Accept: "application/sdp", + "Content-Type": "application/sdp", + Authorization: `Bearer ${token}`, + }, + body: offer.sdp, + }); - const offer = await pc.createOffer(); - await pc.setLocalDescription(offer); + if (response.status !== 201) { + throw Error("Unable to connect to the server"); + } - const response = await fetch(`${url}/api/whip`, { - method: 'POST', - cache: 'no-cache', - headers: { - Accept: 'application/sdp', - 'Content-Type': 'application/sdp', - Authorization: `Bearer ${token}`, - }, - body: offer.sdp, - }); + const sdp = await response.text(); + await pc.setRemoteDescription({ type: "answer", sdp: sdp }); +} - if (response.status !== 201) { - throw Error("Unable to connect to the server"); - } +async function start() { + let browser; + + try { + console.log("Initialising the browser..."); + browser = await puppeteer.launch({ + args: [ + "--no-sandbox", + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + ], + }); + const page = await browser.newPage(); + page.on("console", (msg) => console.log("Page log:", msg.text())); + + // we need a page with secure context in order to access userMedia + await page.goto(`${url}/notfound`); - const sdp = await response.text(); - await pc.setRemoteDescription({ type: 'answer', sdp: sdp }); - }, url, token); + await page.evaluate(stream, url, token); + } catch (err) { + console.error("Browser error occured:", err); + if (browser) await browser.close(); + } } start(); From a743ae09d978bec5d4d494d56aab298f4c5d6ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Fri, 2 Aug 2024 10:48:55 +0200 Subject: [PATCH 6/7] Turn off the logs --- broadcaster/config/config.exs | 2 +- broadcaster/config/prod.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/broadcaster/config/config.exs b/broadcaster/config/config.exs index 3ec89e9..216ffe1 100644 --- a/broadcaster/config/config.exs +++ b/broadcaster/config/config.exs @@ -47,7 +47,7 @@ config :tailwind, config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id], - level: :debug + level: :info # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason diff --git a/broadcaster/config/prod.exs b/broadcaster/config/prod.exs index 62a0773..49d7c24 100644 --- a/broadcaster/config/prod.exs +++ b/broadcaster/config/prod.exs @@ -9,7 +9,7 @@ config :broadcaster, BroadcasterWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json" # Do not print debug messages in production -config :logger, level: :debug +config :logger, level: :info # Runtime production configuration, including reading # of environment variables, is done on config/runtime.exs. From 11f3aa4bdd730230913d03b4d799d1f1cc336885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wala?= Date: Fri, 2 Aug 2024 11:17:11 +0200 Subject: [PATCH 7/7] Handle new way of returning RTCP packets --- broadcaster/lib/broadcaster/forwarder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broadcaster/lib/broadcaster/forwarder.ex b/broadcaster/lib/broadcaster/forwarder.ex index 2235ad9..05c994a 100644 --- a/broadcaster/lib/broadcaster/forwarder.ex +++ b/broadcaster/lib/broadcaster/forwarder.ex @@ -205,7 +205,7 @@ defmodule Broadcaster.Forwarder do def handle_info({:ex_webrtc, _pc, {:rtcp, packets}}, state) do for packet <- packets do case packet do - %ExRTCP.Packet.PayloadFeedback.PLI{} when state.input_pc != nil -> + {_id, %ExRTCP.Packet.PayloadFeedback.PLI{}} when state.input_pc != nil -> layer = default_layer(state) :ok = PeerConnection.send_pli(state.input_pc, state.video_input, layer)