503 service unavailable

[jgnoonan]

When connecting to the frontend service, we are seeing the following error:

frontend/src/frontend.rs:205] get_client_ids_in_call: failed get clients with unexpected status 503 Service Unavailable
[2024-12-26T14:23:10.570Z ERROR calling_frontend::cleaner] failed to get clients from backend: UnexpectedError(failed get clients with unexpected status 503 Service Unavailable

Stack backtrace:
0: anyhow::kind::Adhoc::new
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
1: <calling_frontend::backend::BackendHttpClient as calling_frontend::backend::Backend>::get_clients::{{closure}}
at ./usr/src/calling-service/frontend/src/backend.rs:288:52
2: <core::pin::Pin

as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
3: calling_frontend::cleaner::does_call_exist_on_backend::{{closure}}
at ./usr/src/calling-service/frontend/src/cleaner.rs:36:18
4: calling_frontend::cleaner::start::{{closure}}::{{closure}}::{{closure}}
at ./usr/src/calling-service/frontend/src/cleaner.rs:91:88
5: <futures_util::future::maybe_done::MaybeDone as core::future::future::Future>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/maybe_done.rs:95:38
6: <futures_util::future::join_all::JoinAll as core::future::future::Future>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/join_all.rs:143:24
7: calling_frontend::cleaner::start::{{closure}}::{{closure}}
at ./usr/src/calling-service/frontend/src/cleaner.rs:100:44
8: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
9: tokio::loom::std::unsafe_cell::UnsafeCell::with_mut
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
10: tokio::runtime::task::core::Core<T,S>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30
11: tokio::runtime::task::harness::poll_future::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
12: <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
13: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
14: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
15: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
16: tokio::runtime::task::harness::poll_future
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
17: tokio::runtime::task::harness::Harness<T,S>::poll_inner
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
18: tokio::runtime::task::harness::Harness<T,S>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
19: tokio::runtime::task::raw::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
20: tokio::runtime::task::raw::RawTask::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
21: tokio::runtime::task::LocalNotified::run
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9
22: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:648:22
23: tokio::runtime::coop::with_budget
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5
24: tokio::runtime::coop::budget
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5
25: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9
26: tokio::runtime::scheduler::multi_thread::worker::Context::run
27: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21
28: tokio::runtime::context::scoped::Scoped::set
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9
29: tokio::runtime::context::set_scheduler::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26
30: std::thread::local::LocalKey::try_with
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12
31: std::thread::local::LocalKey::with
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9
32: tokio::runtime::context::set_scheduler
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:17
33: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9
34: tokio::runtime::context::runtime::enter_runtime
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
35: tokio::runtime::scheduler::multi_thread::worker::run
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5
36: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45
37: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21
38: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
39: tokio::loom::std::unsafe_cell::UnsafeCell::with_mut
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
40: tokio::runtime::task::core::Core<T,S>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30
41: tokio::runtime::task::harness::poll_future::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
42: <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
43: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
44: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
45: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
46: tokio::runtime::task::harness::poll_future
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
47: tokio::runtime::task::harness::Harness<T,S>::poll_inner
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
48: tokio::runtime::task::harness::Harness<T,S>::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
49: tokio::runtime::task::raw::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
50: tokio::runtime::task::raw::RawTask::poll
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
51: tokio::runtime::task::UnownedTask::run
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9
52: tokio::runtime::blocking::pool::Task::run
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9
53: tokio::runtime::blocking::pool::Inner::run
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17
54: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
at ./root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13
55: std::sys::backtrace::rust_begin_short_backtrace
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18
56: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17
57: <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
58: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
59: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
60: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
61: std::thread::Builder::spawn_unchecked::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30
62: core::ops::function::FnOnce::call_once{{vtable.shim}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5
63: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
64: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
65: std::sys::pal::unix::thread::Thread::new::thread_start
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17
66: start_thread
67: clone)

The front end is receiving the request, but the backend is never reached. Our docker-compose.yml is below with authentication redacted:

version: '3.8'

services:

dynamodb:
image: "amazon/dynamodb-local:latest"
container_name: dynamodb
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
ports:
- "8000:8000"
working_dir: /home/dynamodblocal
command:
# Use -inMemory for performance and because the data is temporary.
-jar DynamoDBLocal.jar -inMemory

Sets up the table for the Calling Frontend. This sets up a table here:

  #region: us-west-1
#table name: Conferences

bootstrap:
depends_on:
- dynamodb
build: ./docker/bootstrap
container_name: bootstrap
environment:
AWS_ACCESS_KEY_ID: 'DummyAccessKey'
AWS_SECRET_ACCESS_KEY: 'DummyPassword'

calling-frontend:
depends_on:
- dynamodb
- bootstrap
- calling-backend
build:
args:
rust_flags: -Ctarget-cpu=native
dockerfile: frontend/Dockerfile
context: .
container_name: calling-frontend
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
deploy:
resources:
limits:
cpus: '2.0'
memory: 5G
ports:
# Note: The frontend can be accessed via port 8090 externally.
- "8090:8080"
- "8100:8100"
entrypoint: calling_frontend
command:
- --backend-ip
- "127.0.0.1"
- --region
- "us-west1"
- --authentication-key
# Set the auth key to the one you want, but do not commit with it!
- "c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7"
- --zkparams
- "ABLYTlr7IffenO2sRZwt8lJKqCaphrW17Ow5yT2thuYNTs9vWRND4lrZ0RNuivHcOMPbwmhLwB0jPX0I51BwsANgSz/CCC7VV5t3JPZABO+Q3YJtxolvnz6+uYBVczvYWufMzL5IbUAmlzl00i+eN0VCmyzMzNqsYp5bW8EMcP8BEEgghCnrVjSEVC0+9pIhUrxtk10DYRfx7F1uRpQyRweZTXMwaEg+j9yC/Whj+2NYqkP9wqgVd2OVqjaQP9w/BEX3wj8BRRjpKeqCLV12ujWx+4X6TzLa/hyO9OBHN1cDbzqY8SKeAqLbzOBKfDUz5QQu53bNVusM1/Qe754lUAsYNkgsxj3vDIeNoUQ5BNviAfCiWYTAefOtPWUa0XoWDObD+KF94q79LP9DfP6xrMIp032lNfbxBKiLzpjZx7kFy+999HR6NnZoOjulhFDJanboTPUtvq2qb9pdwosz9QSm+AagMqZmI8jLgZv48UEzGGsR3BULCon4zygdvsd/CQ=="
- --version
- "1"
- --max-clients-per-call
- "50"
- --cleanup-interval-ms
- "10000"
- --regional-url-template
- "http://calling-frontend:8080/"
- --calling-server-url
- "http://127.0.0.1:8080/"
- --storage-table
- "Rooms"
- --storage-endpoint
- "http://dynamodb:8000/"
- --storage-region
- "us-west-1"
- --internal-api-port
- "8100"

calling-backend:
build:
args:
rust_flags: -Ctarget-cpu=native
dockerfile: backend/Dockerfile
context: .
container_name: calling-backend
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
deploy:
resources:
limits:
cpus: '4.0'
memory: 5G
ports:
- "8080:8080"
- "9900:9900"
- "10000:10000/udp"
entrypoint: calling_backend
command:
# Set these IPs to your host's IP if you need to access it externally.
- --binding-ip
- "127.0.0.1"
- --ice-candidate-ip
- "127.0.0.1"
- --signaling-ip
- "127.0.0.1"
- --ice-candidate-port-tcp
- "9900"
- --inactivity-timeout-secs
- "60"
- --diagnostics-interval-secs
- "10"
- --approved-users-persistence-url
- "http://calling-frontend:8100/v1/call-link-approvals"
- --remove-call-records-base-url
- "http://calling-frontend:8100/v2/conference"

Go here on the host to find out the ngrok tunnel URL: http://localhost:4040/

ngrok-frontend:
depends_on:
- calling-frontend
image: ngrok/ngrok:latest
ports:
- 4040:4040
# Optional: Set your ngrok auth token here to avoid timeouts and rate limits.
#environment:
# - NGROK_AUTHTOKEN=
command:
- "http"
- "calling-frontend:8080"

networks:
default:
name: test-network

Any assistance with this issue is greatly appreciated.