Merge pull request #97 from arielb1/better-docs

Better docs, also allow spawning its own runtime

Author: rcoh

.github/actions/rust-build/action.yml

@@ -24,4 +24,8 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-          cargo test --verbose ${{ inputs.flags }}
+          cargo test --verbose ${{ inputs.flags }}
+          if [ "${{ inputs.toolchain }}" == nightly -a "${{ inputs.flags }}" == "--all-features" ]; then
+              # docs use unstable features, run them on nightly
+              RUSTDOCFLAGS="-D warnings --cfg docsrs" cargo doc --verbose ${{ inputs.flags }}
+          fi

.github/workflows/build.yml

@@ -10,6 +10,7 @@ jobs:
         toolchain:
           - "1.86" # Current MSRV due to 1.85 having problems with the AWS SDK
           - stable
+          - nightly
         flags:
           - "--all-features"
           - "--no-default-features"
@@ -92,4 +93,4 @@ jobs:
       - name: Run integration test
         shell: bash
         working-directory: tests
-        run: chmod +x simple pollcatch-decoder && LD_LIBRARY_PATH=$PWD ./integration.sh
+        run: chmod +x simple pollcatch-decoder && LD_LIBRARY_PATH=$PWD ./integration.sh && LD_LIBRARY_PATH=$PWD ./separate_runtime_integration.sh

.github/workflows/format.yml

@@ -27,6 +27,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+      - name: Install clippy
+        shell: bash
+        run: rustup component add clippy
       - name: Run clippy check
         id: cargoClippy
         shell: bash
@@ -40,6 +43,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-directories: decoder
+      - name: Install clippy
+        shell: bash
+        run: rustup component add clippy
       - name: Run clippy check - decoder
         id: cargoClippyDecoder
         shell: bash

Cargo.toml

@@ -47,3 +47,8 @@ s3-no-defaults = ["dep:aws-config", "dep:aws-sdk-s3"]
 aws-metadata = ["aws-metadata-no-defaults", "aws-config/default", "reqwest/rustls-tls"]
 # A version of the aws-metadata feature that does not enable AWS default features
 aws-metadata-no-defaults = ["dep:reqwest", "dep:aws-config", "dep:aws-arn"]
+
+[package.metadata.docs.rs]
+all-features = true
+targets = ["x86_64-unknown-linux-gnu"]
+rustdoc-args = ["--cfg", "docsrs"]

README.md

@@ -76,6 +76,35 @@ The metadata is not used by the agent directly, and only provided to the reporte
 [Fargate]: https://aws.amazon.com/fargate
 [IMDS]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
 
+### What information the profiler gathers
+
+Memory samples (JFR `profiler.Malloc`) sample allocated memory every
+so many bytes of allocated memory, and are matched by `profiler.Free`
+to allow detecting if that memory is not free'd.
+
+CPU-time samples (JFR `jdk.ExecutionSample`) sample only threads that
+are currently running on a CPU, not threads that are sleeping.
+
+Wall-clock samples (JFR `profiler.WallClockSample`) sample threads
+whether they are sleeping or running, and can therefore be
+very useful for finding threads that are blocked, for example
+on a synchronous lock or a slow system call.
+
+When using Tokio, since tasks are not threads, tasks that are not
+currently running will not be sampled by a wall clock sample. However,
+a wall clock sample is still very useful in Tokio, since it is what
+you want to catch tasks that are blocking a thread by waiting on
+synchronous operations.
+
+The default is to do a wall-clock sample every second, and a CPU-time
+sample every 100 CPU milliseconds. This can be configured via
+[`ProfilerOptionsBuilder`].
+
+Memory samples are not enabled by default, but can be enabled by [`with_native_mem_bytes`].
+
+[`ProfilerOptionsBuilder`]: https://docs.rs/async-profiler-agent/0.1/async_profiler_agent/profiler/struct.ProfilerOptionsBuilder.html
+[`with_native_mem_bytes`]: https://docs.rs/async-profiler-agent/0.1/async_profiler_agent/profiler/struct.ProfilerOptionsBuilder.html#method.with_native_mem_bytes
+
 ### PollCatch
 
 If you want to find long poll times, and you have `RUSTFLAGS="--cfg tokio_unstable"`, you can

examples/simple/main.rs

@@ -40,6 +40,8 @@ struct S3BucketArgs {
 }
 
 /// Simple program to test the profiler agent
+///
+/// This program is intended for test purposes ONLY.
 #[derive(Debug, Parser)]
 #[command(group(
     ArgGroup::new("options")
@@ -62,6 +64,9 @@ struct Args {
     worker_threads: Option<usize>,
     #[arg(long)]
     native_mem: Option<String>,
+    /// Use the spawn_thread API instead of the Tokio API (does not demonstrate stopping)
+    #[arg(long)]
+    spawn_into_thread: bool,
 }
 
 impl Args {
@@ -95,6 +100,16 @@ pub fn main() -> anyhow::Result<()> {
     rt.block_on(main_internal(args))
 }
 
+async fn run_slow(args: &Args) {
+    if let Some(timeout) = args.duration {
+        tokio::time::timeout(timeout, slow::run())
+            .await
+            .unwrap_err();
+    } else {
+        slow::run().await;
+    }
+}
+
 async fn main_internal(args: Args) -> Result<(), anyhow::Error> {
     set_up_tracing();
     tracing::info!("main started");
@@ -104,7 +119,7 @@ async fn main_internal(args: Args) -> Result<(), anyhow::Error> {
     let profiler = match (&args.local, args.s3_bucket_args()) {
         (Some(local), S3BucketArgs { .. }) => profiler
             .with_reporter(LocalReporter::new(local))
-            .with_custom_agent_metadata(AgentMetadata::Other),
+            .with_custom_agent_metadata(AgentMetadata::NoMetadata),
         #[cfg(feature = "s3-no-defaults")]
         (
             _,
@@ -133,19 +148,21 @@ async fn main_internal(args: Args) -> Result<(), anyhow::Error> {
         .with_profiler_options(profiler_options)
         .build();
 
-    tracing::info!("starting profiler");
-    let handle = profiler.spawn_controllable()?;
-    tracing::info!("profiler started");
-
-    if let Some(timeout) = args.duration {
-        tokio::time::timeout(timeout, slow::run())
-            .await
-            .unwrap_err();
+    if args.spawn_into_thread {
+        tracing::info!("starting profiler");
+        std::thread::spawn(move || {
+            profiler.spawn_thread().unwrap();
+        });
+        run_slow(&args).await;
     } else {
-        slow::run().await;
-    }
+        tracing::info!("starting profiler");
+        let handle = profiler.spawn_controllable()?;
+        tracing::info!("profiler started");
 
-    handle.stop().await;
+        run_slow(&args).await;
+
+        handle.stop().await;
+    }
 
     Ok(())
 }

src/asprof/mod.rs

@@ -108,18 +108,18 @@ impl super::profiler::ProfilerEngine for AsProf {
         jfr_file_path: &Path,
         options: &ProfilerOptions,
     ) -> Result<(), self::AsProfError> {
-        tracing::debug!("starting the async-profiler and giving JFR file path: {jfr_file_path:?}");
+        tracing::debug!("starting profiling session and giving JFR file path: {jfr_file_path:?}");
 
         let args = options.to_args_string(jfr_file_path);
 
         Self::asprof_execute(&args)?;
-        tracing::debug!("async-profiler started successfully");
+        tracing::debug!("starting profiling session - success");
         Ok(())
     }
 
     fn stop_async_profiler() -> Result<(), self::AsProfError> {
         Self::asprof_execute("stop")?;
-        tracing::debug!("async-profiler stopped successfully");
+        tracing::debug!("stopping profiling session - success");
         Ok(())
     }
 }

src/lib.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #![deny(missing_docs)]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
 
 //! ## async-profiler Rust agent
 //! An in-process Rust agent for profiling an application using [async-profiler] and uploading the resulting profiles.

src/metadata/mod.rs

@@ -6,7 +6,7 @@
 pub use std::time::Duration;
 
 /// Host Metadata, which describes a host that runs a profiling agent. The current set of supported agent metadata is
-/// AWS-specific. If you are not running on AWS, you can use [AgentMetadata::Other].
+/// AWS-specific. If you are not running on AWS, you can use [AgentMetadata::NoMetadata].
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum AgentMetadata {
@@ -43,7 +43,11 @@ pub enum AgentMetadata {
         ecs_cluster_arn: String,
     },
     /// Metadata for a host that is neither an EC2 nor a Fargate
+    #[deprecated = "Use AgentMetadata::NoMetadata"]
     Other,
+    /// A placeholder when a host has no metadata, or when a reporter does not
+    /// use metadata.
+    NoMetadata,
 }
 
 /// Metadata associated with a specific individual profiling report
@@ -66,7 +70,7 @@ pub mod aws;
 /// [private] dummy metadata to make testing easier
 #[cfg(test)]
 pub(crate) const DUMMY_METADATA: ReportMetadata<'static> = ReportMetadata {
-    instance: &AgentMetadata::Other,
+    instance: &AgentMetadata::NoMetadata,
     start: Duration::from_secs(1),
     end: Duration::from_secs(2),
     reporting_interval: Duration::from_secs(1),