add delayed start to heartbeats (#387)

Adds a random initial jitter the size of the heartbeat periodicity to prevent heartbeats storming the service when we launch 3000 nodes roughly at the same time.

Fixes #386

src/agent/onefuzz/src/heartbeat.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::utils::CheckNotify;
+use crate::{jitter::random_delay, utils::CheckNotify};
 use anyhow::Result;
 use futures::Future;
 use reqwest::Url;
@@ -83,6 +83,7 @@ where
         TContext: Send + Sync + 'static,
     {
         let heartbeat_period = heartbeat_period.unwrap_or(DEFAULT_HEARTBEAT_PERIOD);
+
         let context = Arc::new(HeartbeatContext {
             state: context,
             queue_client: QueueClient::new(queue_url),
@@ -92,6 +93,7 @@ where
 
         let flush_context = context.clone();
         let heartbeat_process = task::spawn(async move {
+            random_delay(heartbeat_period).await;
             flush(flush_context.clone()).await;
             while !flush_context.cancelled.is_notified(heartbeat_period).await {
                 flush(flush_context.clone()).await;

src/agent/onefuzz/src/jitter.rs

@@ -13,3 +13,9 @@ pub fn jitter(value: Duration) -> Duration {
 pub async fn delay_with_jitter(value: Duration) {
     delay_for(jitter(value)).await
 }
+
+pub async fn random_delay(value: Duration) {
+    let random: u64 = thread_rng().gen_range(0, value.as_secs());
+    let delay = Duration::new(random, 0);
+    delay_for(delay).await
+}