feat(iroh): publish and resolve user-defined data in discovery (#3176)

## Description

**Based on #3175**

Adds support for publishing and resolving a string of user-defined data
through Iroh's discovery services.

This gives applications a way to add additional information to the
records published and resolved through discovery, in addition to the
addressing information (relay URL and direct addresses). Iroh itself
does not parse or use this user-defined data string in any way.

All existing discovery services are updated to support the user data.
All DNS-based discovery services encode the user-defined data in a TXT
record, in the same way as the DNS/pkarr services already encode the
relay URL and direct addresses. Therefore, the length of the
`user-data=<user-data>` string is limited to 255 bytes, which is the max
length for a single character string in DNS messages. Thus, the length
of the actual user-defined data string is limited to 245 bytes (255
minux `user-data=`). This limit is enforced when constructing a
`UserData` (which is just a wrapper around a `String` otherwise).

The local swarm discovery uses `swarm-discovery` under the hood, for
which I added support for TXT records in
rkuhn/swarm-discovery#12. This was released as
`swarm-discovery@0.3.0-alpha.2`.


## Breaking Changes

<!-- Optional, if there are any breaking changes document them,
including how to migrate older code. -->

## Notes & open questions

Note that currently the `DiscoveryItem` which contains the user-defined
data is only accessible when directly calling the methods on the
`Discovery` trait. For discoveries initiated by the endpoint, both
active from `Endpoint::connect` and passive from the magicsock's
subscription to `Discovery::subscribe`, the `DiscoveryItem`s are
currently not exposed in any way, thus there's no way for an app to
access the user data of nodes discovered by the endpoint. However, with
#3181, there will be a way to watch for all `DiscoveryItem`s found by
the endpoint.

## Change checklist

- [x] Self-review.
- [x] Documentation updates following the [style
guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text),
if relevant.
- [x] Tests if relevant.
- [x] All breaking changes documented.

---------

Co-authored-by: Philipp Krüger <philipp.krueger1@gmail.com>

Author: Frando

iroh-dns-server/examples/publish.rs

@@ -1,11 +1,12 @@
-use std::str::FromStr;
+use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::{bail, Result};
 use clap::{Parser, ValueEnum};
 use iroh::{
     discovery::{
         dns::{N0_DNS_NODE_ORIGIN_PROD, N0_DNS_NODE_ORIGIN_STAGING},
         pkarr::{PkarrRelayClient, N0_DNS_PKARR_RELAY_PROD, N0_DNS_PKARR_RELAY_STAGING},
+        UserData,
     },
     dns::node_info::{NodeIdExt, NodeInfo, IROH_TXT_NAME},
     NodeId, SecretKey,
@@ -39,7 +40,14 @@ struct Cli {
     #[clap(long, conflicts_with = "env")]
     pkarr_relay: Option<Url>,
     /// Home relay server to publish for this node
-    relay_url: Url,
+    #[clap(short, long)]
+    relay_url: Option<Url>,
+    /// Direct addresses to publish for this node
+    #[clap(short, long)]
+    addr: Vec<SocketAddr>,
+    /// User data to publish for this node
+    #[clap(short, long)]
+    user_data: Option<UserData>,
     /// Create a new node secret if IROH_SECRET is unset. Only for development / debugging.
     #[clap(short, long)]
     create: bool,
@@ -72,12 +80,23 @@ async fn main() -> Result<()> {
     };
 
     println!("announce {node_id}:");
-    println!("    relay={}", args.relay_url);
+    if let Some(relay_url) = &args.relay_url {
+        println!("    relay={relay_url}");
+    }
+    for addr in &args.addr {
+        println!("    addr={addr}");
+    }
+    if let Some(user_data) = &args.user_data {
+        println!("    user-data={user_data}");
+    }
     println!();
     println!("publish to {pkarr_relay} ...");
 
     let pkarr = PkarrRelayClient::new(pkarr_relay);
-    let node_info = NodeInfo::new(node_id).with_relay_url(Some(args.relay_url.into()));
+    let node_info = NodeInfo::new(node_id)
+        .with_relay_url(args.relay_url.map(Into::into))
+        .with_direct_addresses(args.addr.into_iter().collect())
+        .with_user_data(args.user_data);
     let signed_packet = node_info.to_pkarr_signed_packet(&secret_key, 30)?;
     pkarr.publish(&signed_packet).await?;
 

iroh-dns-server/examples/resolve.rs

@@ -57,5 +57,8 @@ async fn main() -> anyhow::Result<()> {
     for addr in resolved.direct_addresses() {
         println!("    addr={addr}")
     }
+    if let Some(user_data) = resolved.user_data() {
+        println!("    user-data={user_data}")
+    }
     Ok(())
 }

iroh-relay/src/dns/node_info.rs

@@ -34,7 +34,7 @@
 
 use std::{
     collections::{BTreeMap, BTreeSet},
-    fmt::Display,
+    fmt::{self, Display},
     hash::Hash,
     net::SocketAddr,
     str::FromStr,
@@ -80,7 +80,9 @@ impl NodeIdExt for NodeId {
 
 /// Data about a node that may be published to and resolved from discovery services.
 ///
-/// This includes an optional [`RelayUrl`] and a set of direct addresses.
+/// This includes an optional [`RelayUrl`], a set of direct addresses, and the optional
+/// [`UserData`], a string that can be set by applications and is not parsed or used by iroh
+/// itself.
 ///
 /// This struct does not include the node's [`NodeId`], only the data *about* a certain
 /// node. See [`NodeInfo`] for a struct that contains a [`NodeId`] with associated [`NodeData`].
@@ -90,6 +92,8 @@ pub struct NodeData {
     relay_url: Option<RelayUrl>,
     /// Direct addresses where this node can be reached.
     direct_addresses: BTreeSet<SocketAddr>,
+    /// Optional user-defined [`UserData`] for this node.
+    user_data: Option<UserData>,
 }
 
 impl NodeData {
@@ -98,6 +102,7 @@ impl NodeData {
         Self {
             relay_url,
             direct_addresses,
+            user_data: None,
         }
     }
 
@@ -113,11 +118,22 @@ impl NodeData {
         self
     }
 
+    /// Sets the user-defined data and returns the updated node data.
+    pub fn with_user_data(mut self, user_data: Option<UserData>) -> Self {
+        self.user_data = user_data;
+        self
+    }
+
     /// Returns the relay URL of the node.
     pub fn relay_url(&self) -> Option<&RelayUrl> {
         self.relay_url.as_ref()
     }
 
+    /// Returns the optional user-defined data of the node.
+    pub fn user_data(&self) -> Option<&UserData> {
+        self.user_data.as_ref()
+    }
+
     /// Returns the direct addresses of the node.
     pub fn direct_addresses(&self) -> &BTreeSet<SocketAddr> {
         &self.direct_addresses
@@ -137,17 +153,84 @@ impl NodeData {
     pub fn set_relay_url(&mut self, relay_url: Option<RelayUrl>) {
         self.relay_url = relay_url
     }
+
+    /// Sets the user-defined data of the node data.
+    pub fn set_user_data(&mut self, user_data: Option<UserData>) {
+        self.user_data = user_data;
+    }
 }
 
 impl From<NodeAddr> for NodeData {
     fn from(node_addr: NodeAddr) -> Self {
         Self {
             relay_url: node_addr.relay_url,
             direct_addresses: node_addr.direct_addresses,
+            user_data: None,
+        }
+    }
+}
+
+// User-defined data that can be published and resolved through node discovery.
+///
+/// Under the hood this is a UTF-8 String is no longer than [`UserData::MAX_LENGTH`] bytes.
+///
+/// Iroh does not keep track of or examine the user-defined data.
+///
+/// `UserData` implements [`FromStr`] and [`TryFrom<String>`], so you can
+/// convert `&str` and `String` into `UserData` easily.
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub struct UserData(String);
+
+impl UserData {
+    /// The max byte length allowed for user-defined data.
+    ///
+    /// In DNS discovery services, the user-defined data is stored in a TXT record character string,
+    /// which has a max length of 255 bytes. We need to subtract the `user-data=` prefix,
+    /// which leaves 245 bytes for the actual user-defined data.
+    pub const MAX_LENGTH: usize = 245;
+}
+
+/// Error returned when an input value is too long for [`UserData`].
+#[derive(Debug, thiserror::Error)]
+#[error("User-defined data exceeds max length")]
+pub struct MaxLengthExceededError;
+
+impl TryFrom<String> for UserData {
+    type Error = MaxLengthExceededError;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        if value.len() > Self::MAX_LENGTH {
+            Err(MaxLengthExceededError)
+        } else {
+            Ok(Self(value))
+        }
+    }
+}
+
+impl FromStr for UserData {
+    type Err = MaxLengthExceededError;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        if s.len() > Self::MAX_LENGTH {
+            Err(MaxLengthExceededError)
+        } else {
+            Ok(Self(s.to_string()))
         }
     }
 }
 
+impl fmt::Display for UserData {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl AsRef<str> for UserData {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
 /// Information about a node that may be published to and resolved from discovery services.
 ///
 /// This struct couples a [`NodeId`] with its associated [`NodeData`].
@@ -181,9 +264,16 @@ impl From<&TxtAttrs<IrohAttr>> for NodeInfo {
             .flatten()
             .filter_map(|s| SocketAddr::from_str(s).ok())
             .collect();
+        let user_data = attrs
+            .get(&IrohAttr::UserData)
+            .into_iter()
+            .flatten()
+            .next()
+            .and_then(|s| UserData::from_str(s).ok());
         let data = NodeData {
             relay_url: relay_url.map(Into::into),
             direct_addresses,
+            user_data,
         };
         Self { node_id, data }
     }
@@ -226,6 +316,12 @@ impl NodeInfo {
         self
     }
 
+    /// Sets the user-defined data and returns the updated node info.
+    pub fn with_user_data(mut self, user_data: Option<UserData>) -> Self {
+        self.data = self.data.with_user_data(user_data);
+        self
+    }
+
     /// Converts into a [`NodeAddr`] by cloning the needed fields.
     pub fn to_node_addr(&self) -> NodeAddr {
         NodeAddr {
@@ -321,6 +417,8 @@ pub(super) enum IrohAttr {
     Relay,
     /// Direct address.
     Addr,
+    /// User-defined data
+    UserData,
 }
 
 /// Attributes parsed from [`IROH_TXT_NAME`] TXT records.
@@ -343,6 +441,9 @@ impl From<&NodeInfo> for TxtAttrs<IrohAttr> {
         for addr in &info.data.direct_addresses {
             attrs.push((IrohAttr::Addr, addr.to_string()));
         }
+        if let Some(user_data) = &info.data.user_data {
+            attrs.push((IrohAttr::UserData, user_data.to_string()));
+        }
         Self::from_parts(info.node_id, attrs.into_iter())
     }
 }
@@ -552,7 +653,8 @@ mod tests {
         let node_data = NodeData::new(
             Some("https://example.com".parse().unwrap()),
             ["127.0.0.1:1234".parse().unwrap()].into_iter().collect(),
-        );
+        )
+        .with_user_data(Some("foobar".parse().unwrap()));
         let node_id = "vpnk377obfvzlipnsfbqba7ywkkenc4xlpmovt5tsfujoa75zqia"
             .parse()
             .unwrap();
@@ -569,7 +671,8 @@ mod tests {
         let node_data = NodeData::new(
             Some("https://example.com".parse().unwrap()),
             ["127.0.0.1:1234".parse().unwrap()].into_iter().collect(),
-        );
+        )
+        .with_user_data(Some("foobar".parse().unwrap()));
         let expected = NodeInfo::from_parts(secret_key.public(), node_data);
         let packet = expected.to_pkarr_signed_packet(&secret_key, 30).unwrap();
         let actual = NodeInfo::from_pkarr_signed_packet(&packet).unwrap();

iroh/src/discovery.rs

@@ -109,7 +109,7 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, ensure, Result};
 use iroh_base::{NodeAddr, NodeId};
-pub use iroh_relay::dns::node_info::{NodeData, NodeInfo};
+pub use iroh_relay::dns::node_info::{NodeData, NodeInfo, UserData};
 use n0_future::{
     stream::{Boxed as BoxStream, StreamExt},
     task::{self, AbortOnDropHandle},
@@ -943,7 +943,7 @@ mod tests {
 mod test_dns_pkarr {
     use anyhow::Result;
     use iroh_base::{NodeAddr, SecretKey};
-    use iroh_relay::RelayMap;
+    use iroh_relay::{dns::node_info::UserData, RelayMap};
     use n0_future::time::Duration;
     use tokio_util::task::AbortOnDropHandle;
     use tracing_test::traced_test;
@@ -996,20 +996,24 @@ mod test_dns_pkarr {
 
         let resolver = DnsResolver::with_nameserver(dns_pkarr_server.nameserver);
         let publisher = PkarrPublisher::new(secret_key, dns_pkarr_server.pkarr_url.clone());
-        let data = NodeData::new(relay_url.clone(), Default::default());
+        let user_data: UserData = "foobar".parse().unwrap();
+        let data = NodeData::new(relay_url.clone(), Default::default())
+            .with_user_data(Some(user_data.clone()));
         // does not block, update happens in background task
         publisher.update_node_data(&data);
         // wait until our shared state received the update from pkarr publishing
         dns_pkarr_server.on_node(&node_id, PUBLISH_TIMEOUT).await?;
         let resolved = resolver.lookup_node_by_id(&node_id, &origin).await?;
+        println!("resolved {resolved:?}");
 
-        let expected = NodeAddr {
+        let expected_addr = NodeAddr {
             node_id,
             relay_url,
             direct_addresses: Default::default(),
         };
 
-        assert_eq!(resolved.to_node_addr(), expected);
+        assert_eq!(resolved.to_node_addr(), expected_addr);
+        assert_eq!(resolved.user_data(), Some(&user_data));
         Ok(())
     }