Skip to content

Commit 9a9190b

Browse files
committed
feat(runtime): Support tokio-console
See https://github.com/tokio-rs/console for details. Usage: - Build: `cargo build --features cuda,tokio-console -p dynamo-run` - Start dynamo-run - `cargo install --locked tokio-console` - `tokio-console`
1 parent 49b7a0d commit 9a9190b

File tree

8 files changed

+147
-94
lines changed

8 files changed

+147
-94
lines changed

.cargo/config.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[build]
2+
# tokio-console needs this
3+
rustflags = ["--cfg", "tokio_unstable"]

Cargo.lock

Lines changed: 60 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
# SPDX-License-Identifier: Apache-2.0
3-
#
4-
# Licensed under the Apache License, Version 2.0 (the "License");
5-
# you may not use this file except in compliance with the License.
6-
# You may obtain a copy of the License at
7-
#
8-
# http://www.apache.org/licenses/LICENSE-2.0
9-
#
10-
# Unless required by applicable law or agreed to in writing, software
11-
# distributed under the License is distributed on an "AS IS" BASIS,
12-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
# See the License for the specific language governing permissions and
14-
# limitations under the License.
153

164
[workspace]
175
members = [

docs/guides/dynamo_run.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -669,3 +669,12 @@ More fully-featured Backend engines (used by `dynamo-run`):
669669
- [vllm](https://github.com/ai-dynamo/dynamo/blob/main/launch/dynamo-run/src/subprocess/vllm_inc.py)
670670
- [sglang](https://github.com/ai-dynamo/dynamo/blob/main/launch/dynamo-run/src/subprocess/sglang_inc.py)
671671

672+
### Debugging
673+
674+
`dynamo-run` and `dynamo-runtime` support [tokio-console](https://github.com/tokio-rs/console). Build with the feature to enable:
675+
```
676+
cargo build --features cuda,tokio-console -p dynamo-run
677+
```
678+
679+
The listener uses the default tokio console port, and all interfaces (0.0.0.0).
680+

launch/dynamo-run/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ metal = ["dynamo-engine-llamacpp/metal", "dynamo-engine-mistralrs/metal"]
2323
vulkan = ["dynamo-engine-llamacpp/vulkan"]
2424
openmp = ["dynamo-engine-llamacpp/openmp"]
2525

26+
tokio-console = ["dynamo-runtime/tokio-console"]
27+
2628
[dependencies]
2729
dynamo-llm = { workspace = true }
2830
dynamo-runtime = { workspace = true }

launch/dynamo-run/src/main.rs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
3-
//
4-
// Licensed under the Apache License, Version 2.0 (the "License");
5-
// you may not use this file except in compliance with the License.
6-
// You may obtain a copy of the License at
7-
//
8-
// http://www.apache.org/licenses/LICENSE-2.0
9-
//
10-
// Unless required by applicable law or agreed to in writing, software
11-
// distributed under the License is distributed on an "AS IS" BASIS,
12-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
// See the License for the specific language governing permissions and
14-
// limitations under the License.
153

164
use std::env;
175

lib/runtime/Cargo.toml

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
# SPDX-License-Identifier: Apache-2.0
3-
#
4-
# Licensed under the Apache License, Version 2.0 (the "License");
5-
# you may not use this file except in compliance with the License.
6-
# You may obtain a copy of the License at
7-
#
8-
# http://www.apache.org/licenses/LICENSE-2.0
9-
#
10-
# Unless required by applicable law or agreed to in writing, software
11-
# distributed under the License is distributed on an "AS IS" BASIS,
12-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
# See the License for the specific language governing permissions and
14-
# limitations under the License.
153

164
[package]
175
name = "dynamo-runtime"
@@ -27,8 +15,8 @@ description = "Dynamo Runtime Library"
2715
[features]
2816
default = []
2917
integration = []
30-
# Tests that require an active ETCD server
31-
testing-etcd = []
18+
testing-etcd = [] # Tests that require an active ETCD server
19+
tokio-console = ["dep:console-subscriber", "tokio/tracing"]
3220

3321
[dependencies]
3422
# Use workspace dependencies where available
@@ -64,6 +52,7 @@ xxhash-rust = { workspace = true }
6452

6553
arc-swap = { version = "1" }
6654
async-once-cell = { version = "0.5.4" }
55+
console-subscriber = { version = "0.4", optional = true }
6756
educe = { version = "0.6.0" }
6857
figment = { version = "0.10.19", features = ["env", "json", "toml", "test"] }
6958
local-ip-address = { version = "0.6.3" }
@@ -81,10 +70,3 @@ env_logger = { version = "0.11" }
8170
reqwest = { workspace = true }
8271
rstest = { version = "0.23.0" }
8372
temp-env = { version = "0.3.6" }
84-
85-
# These patches are to address issues in reqwest, which is used in the HTTP server test (but not on servers).
86-
# These are transitive dependencies to use secure versions and mitigate known vulnerabilities.
87-
[patch.crates-io]
88-
tokio = { version = "1.18.4" } # addresses RUSTSEC-2023-0001
89-
h2 = { version = "0.4.4" } # addresses RUSTSEC-2024-0332
90-
rustls = { version = "0.23.18" } # addresses RUSTSEC-2024-0399

lib/runtime/src/logging.rs

Lines changed: 70 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
3-
//
4-
// Licensed under the Apache License, Version 2.0 (the "License");
5-
// you may not use this file except in compliance with the License.
6-
// You may obtain a copy of the License at
7-
//
8-
// http://www.apache.org/licenses/LICENSE-2.0
9-
//
10-
// Unless required by applicable law or agreed to in writing, software
11-
// distributed under the License is distributed on an "AS IS" BASIS,
12-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
// See the License for the specific language governing permissions and
14-
// limitations under the License.
153

164
//! Dynamo Distributed Logging Module.
175
//!
@@ -46,6 +34,7 @@ use figment::{
4634
Figment,
4735
};
4836
use serde::{Deserialize, Serialize};
37+
use tracing::level_filters::LevelFilter;
4938
use tracing::{Event, Subscriber};
5039
use tracing_subscriber::fmt::time::FormatTime;
5140
use tracing_subscriber::fmt::time::LocalTime;
@@ -58,6 +47,8 @@ use tracing_subscriber::registry::LookupSpan;
5847
use tracing_subscriber::EnvFilter;
5948
use tracing_subscriber::{filter::Directive, fmt};
6049

50+
use crate::config::{disable_ansi_logging, jsonl_logging_enabled};
51+
6152
/// ENV used to set the log level
6253
const FILTER_ENV: &str = "DYN_LOG";
6354

@@ -98,45 +89,75 @@ impl Default for LoggingConfig {
9889

9990
/// Initialize the logger
10091
pub fn init() {
101-
INIT.call_once(|| {
102-
let config = load_config();
103-
104-
// Examples to remove noise
105-
// .add_directive("rustls=warn".parse()?)
106-
// .add_directive("tokio_util::codec=warn".parse()?)
107-
let mut filter_layer = EnvFilter::builder()
108-
.with_default_directive(config.log_level.parse().unwrap())
109-
.with_env_var(FILTER_ENV)
110-
.from_env_lossy();
111-
112-
// apply the log_filters from the config files
113-
for (module, level) in config.log_filters {
114-
match format!("{module}={level}").parse::<Directive>() {
115-
Ok(d) => {
116-
filter_layer = filter_layer.add_directive(d);
117-
}
118-
Err(e) => {
119-
eprintln!("Failed parsing filter '{level}' for module '{module}': {e}");
120-
}
92+
INIT.call_once(setup_logging);
93+
}
94+
95+
#[cfg(feature = "tokio-console")]
96+
fn setup_logging() {
97+
// Start tokio-console server. Returns a tracing-subscriber Layer.
98+
let tokio_console_layer = console_subscriber::ConsoleLayer::builder()
99+
.with_default_env()
100+
.server_addr(([0, 0, 0, 0], console_subscriber::Server::DEFAULT_PORT))
101+
.spawn();
102+
let tokio_console_target = tracing_subscriber::filter::Targets::new()
103+
.with_default(LevelFilter::ERROR)
104+
.with_target("runtime", LevelFilter::TRACE)
105+
.with_target("tokio", LevelFilter::TRACE);
106+
let l = fmt::layer()
107+
.with_ansi(!disable_ansi_logging())
108+
.event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
109+
.with_writer(std::io::stderr)
110+
.with_filter(filter(load_config));
111+
tracing_subscriber::registry()
112+
.with(l)
113+
.with(tokio_console_layer.with_filter(tokio_console_target))
114+
.init();
115+
}
116+
117+
#[cfg(not(feature = "tokio-console"))]
118+
fn setup_logging() {
119+
let f = filters(load_config());
120+
// The generics mean we have to repeat everything. Each builder method returns a
121+
// specialized type.
122+
if jsonl_logging_enabled() {
123+
// JSON logger for NIM
124+
125+
let l = fmt::layer()
126+
.with_ansi(false)
127+
.event_format(CustomJsonFormatter::new())
128+
.with_writer(std::io::stderr)
129+
.with_filter(f);
130+
tracing_subscriber::registry().with(l).init();
131+
} else {
132+
// Normal logging
133+
134+
let l = fmt::layer()
135+
.with_ansi(!disable_ansi_logging())
136+
.event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
137+
.with_writer(std::io::stderr)
138+
.with_filter(f);
139+
tracing_subscriber::registry().with(l).init();
140+
}
141+
}
142+
143+
fn filters(config: LoggingConfig) -> EnvFilter {
144+
let mut filter_layer = EnvFilter::builder()
145+
.with_default_directive(config.log_level.parse().unwrap())
146+
.with_env_var(FILTER_ENV)
147+
.from_env_lossy();
148+
149+
// apply the log_filters from the config files
150+
for (module, level) in config.log_filters {
151+
match format!("{module}={level}").parse::<Directive>() {
152+
Ok(d) => {
153+
filter_layer = filter_layer.add_directive(d);
154+
}
155+
Err(e) => {
156+
eprintln!("Failed parsing filter '{level}' for module '{module}': {e}");
121157
}
122158
}
123-
124-
if crate::config::jsonl_logging_enabled() {
125-
let l = fmt::layer()
126-
.with_ansi(false) // ansi terminal escapes and colors always disabled
127-
.event_format(CustomJsonFormatter::new())
128-
.with_writer(std::io::stderr)
129-
.with_filter(filter_layer);
130-
tracing_subscriber::registry().with(l).init();
131-
} else {
132-
let l = fmt::layer()
133-
.with_ansi(!crate::config::disable_ansi_logging())
134-
.event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
135-
.with_writer(std::io::stderr)
136-
.with_filter(filter_layer);
137-
tracing_subscriber::registry().with(l).init();
138-
};
139-
});
159+
}
160+
filter_layer
140161
}
141162

142163
/// Log a message with file and line info

0 commit comments

Comments
 (0)