Merge pull request #2314 from grumbach/private_data

feat: private data, private archives, vault support and CLI integration

.github/workflows/memcheck.yml

@@ -70,7 +70,7 @@ jobs:
         shell: bash
 
       - name: File upload
-        run: ./target/release/autonomi --log-output-dest=data-dir file upload "./the-test-data.zip" > ./upload_output 2>&1
+        run: ./target/release/autonomi --log-output-dest=data-dir file upload --public "./the-test-data.zip" > ./upload_output 2>&1
         env:
           SN_LOG: "v"
         timeout-minutes: 5

autonomi-cli/Cargo.toml

@@ -49,6 +49,8 @@ sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.5.4" }
 sn_build_info = { path = "../sn_build_info", version = "0.1.16" }
 sn_logging = { path = "../sn_logging", version = "0.2.37" }
 walkdir = "2.5.0"
+serde_json = "1.0.132"
+serde = "1.0.210"
 
 [dev-dependencies]
 autonomi = { path = "../autonomi", version = "0.2.1", features = [

autonomi-cli/src/access/user_data.rs

@@ -11,6 +11,7 @@ use std::collections::HashMap;
 use autonomi::client::{
     address::{addr_to_str, str_to_addr},
     archive::ArchiveAddr,
+    archive_private::PrivateArchiveAccess,
     registers::{RegisterAddress, RegisterSecretKey},
     vault::UserData,
 };
@@ -21,19 +22,62 @@ use super::{
     keys::{create_register_signing_key_file, get_register_signing_key},
 };
 
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+struct PrivateFileArchive {
+    name: String,
+    secret_access: String,
+}
+
 pub fn get_local_user_data() -> Result<UserData> {
     let register_sk = get_register_signing_key().map(|k| k.to_hex()).ok();
     let registers = get_local_registers()?;
-    let file_archives = get_local_file_archives()?;
+    let file_archives = get_local_public_file_archives()?;
+    let private_file_archives = get_local_private_file_archives()?;
 
     let user_data = UserData {
         register_sk,
         registers,
         file_archives,
+        private_file_archives,
     };
     Ok(user_data)
 }
 
+pub fn get_local_private_file_archives() -> Result<HashMap<PrivateArchiveAccess, String>> {
+    let data_dir = get_client_data_dir_path()?;
+    let user_data_path = data_dir.join("user_data");
+    let private_file_archives_path = user_data_path.join("private_file_archives");
+    std::fs::create_dir_all(&private_file_archives_path)?;
+
+    let mut private_file_archives = HashMap::new();
+    for entry in walkdir::WalkDir::new(private_file_archives_path)
+        .min_depth(1)
+        .max_depth(1)
+    {
+        let entry = entry?;
+        let file_content = std::fs::read_to_string(entry.path())?;
+        let private_file_archive: PrivateFileArchive = serde_json::from_str(&file_content)?;
+        let private_file_archive_access =
+            PrivateArchiveAccess::from_hex(&private_file_archive.secret_access)?;
+        private_file_archives.insert(private_file_archive_access, private_file_archive.name);
+    }
+    Ok(private_file_archives)
+}
+
+pub fn get_local_private_archive_access(local_addr: &str) -> Result<PrivateArchiveAccess> {
+    let data_dir = get_client_data_dir_path()?;
+    let user_data_path = data_dir.join("user_data");
+    let private_file_archives_path = user_data_path.join("private_file_archives");
+    let file_path = private_file_archives_path.join(local_addr);
+    let file_content = std::fs::read_to_string(file_path)?;
+    let private_file_archive: PrivateFileArchive = serde_json::from_str(&file_content)?;
+    let private_file_archive_access =
+        PrivateArchiveAccess::from_hex(&private_file_archive.secret_access)?;
+    Ok(private_file_archive_access)
+}
+
 pub fn get_local_registers() -> Result<HashMap<RegisterAddress, String>> {
     let data_dir = get_client_data_dir_path()?;
     let user_data_path = data_dir.join("user_data");
@@ -55,7 +99,7 @@ pub fn get_local_registers() -> Result<HashMap<RegisterAddress, String>> {
     Ok(registers)
 }
 
-pub fn get_local_file_archives() -> Result<HashMap<ArchiveAddr, String>> {
+pub fn get_local_public_file_archives() -> Result<HashMap<ArchiveAddr, String>> {
     let data_dir = get_client_data_dir_path()?;
     let user_data_path = data_dir.join("user_data");
     let file_archives_path = user_data_path.join("file_archives");
@@ -86,8 +130,13 @@ pub fn write_local_user_data(user_data: &UserData) -> Result<()> {
     }
 
     for (archive, name) in user_data.file_archives.iter() {
-        write_local_file_archive(archive, name)?;
+        write_local_public_file_archive(addr_to_str(*archive), name)?;
+    }
+
+    for (archive, name) in user_data.private_file_archives.iter() {
+        write_local_private_file_archive(archive.to_hex(), archive.address(), name)?;
     }
+
     Ok(())
 }
 
@@ -100,11 +149,29 @@ pub fn write_local_register(register: &RegisterAddress, name: &str) -> Result<()
     Ok(())
 }
 
-pub fn write_local_file_archive(archive: &ArchiveAddr, name: &str) -> Result<()> {
+pub fn write_local_public_file_archive(archive: String, name: &str) -> Result<()> {
     let data_dir = get_client_data_dir_path()?;
     let user_data_path = data_dir.join("user_data");
     let file_archives_path = user_data_path.join("file_archives");
     std::fs::create_dir_all(&file_archives_path)?;
-    std::fs::write(file_archives_path.join(addr_to_str(*archive)), name)?;
+    std::fs::write(file_archives_path.join(archive), name)?;
+    Ok(())
+}
+
+pub fn write_local_private_file_archive(
+    archive: String,
+    local_addr: String,
+    name: &str,
+) -> Result<()> {
+    let data_dir = get_client_data_dir_path()?;
+    let user_data_path = data_dir.join("user_data");
+    let private_file_archives_path = user_data_path.join("private_file_archives");
+    std::fs::create_dir_all(&private_file_archives_path)?;
+    let file_name = local_addr;
+    let content = serde_json::to_string(&PrivateFileArchive {
+        name: name.to_string(),
+        secret_access: archive,
+    })?;
+    std::fs::write(private_file_archives_path.join(file_name), content)?;
     Ok(())
 }

autonomi-cli/src/actions/download.rs

@@ -7,12 +7,84 @@
 // permissions and limitations relating to use of the SAFE Network Software.
 
 use super::get_progress_bar;
-use autonomi::{client::address::str_to_addr, Client};
-use color_eyre::eyre::{eyre, Context, Result};
+use autonomi::{
+    client::{address::str_to_addr, archive::ArchiveAddr, archive_private::PrivateArchiveAccess},
+    Client,
+};
+use color_eyre::{
+    eyre::{eyre, Context, Result},
+    Section,
+};
 use std::path::PathBuf;
 
 pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Result<()> {
-    let address = str_to_addr(addr).wrap_err("Failed to parse data address")?;
+    let public_address = str_to_addr(addr).ok();
+    let private_address = crate::user_data::get_local_private_archive_access(addr)
+        .inspect_err(|e| error!("Failed to get private archive access: {e}"))
+        .ok();
+
+    match (public_address, private_address) {
+        (Some(public_address), _) => download_public(addr, public_address, dest_path, client).await,
+        (_, Some(private_address)) => download_private(addr, private_address, dest_path, client).await,
+        _ => Err(eyre!("Failed to parse data address {addr}"))
+            .with_suggestion(|| "Public addresses look like this: 0037cfa13eae4393841cbc00c3a33cade0f98b8c1f20826e5c51f8269e7b09d7")
+            .with_suggestion(|| "Private addresses look like this: 1358645341480028172")
+            .with_suggestion(|| "Try the `file list` command to get addresses you have access to"),
+    }
+}
+
+async fn download_private(
+    addr: &str,
+    private_address: PrivateArchiveAccess,
+    dest_path: &str,
+    client: &mut Client,
+) -> Result<()> {
+    let archive = client
+        .private_archive_get(private_address)
+        .await
+        .wrap_err("Failed to fetch data from address")?;
+
+    let progress_bar = get_progress_bar(archive.iter().count() as u64)?;
+    let mut all_errs = vec![];
+    for (path, access, _meta) in archive.iter() {
+        progress_bar.println(format!("Fetching file: {path:?}..."));
+        let bytes = match client.private_data_get(access.clone()).await {
+            Ok(bytes) => bytes,
+            Err(e) => {
+                let err = format!("Failed to fetch file {path:?}: {e}");
+                all_errs.push(err);
+                continue;
+            }
+        };
+
+        let path = PathBuf::from(dest_path).join(path);
+        let here = PathBuf::from(".");
+        let parent = path.parent().unwrap_or_else(|| &here);
+        std::fs::create_dir_all(parent)?;
+        std::fs::write(path, bytes)?;
+        progress_bar.clone().inc(1);
+    }
+    progress_bar.finish_and_clear();
+
+    if all_errs.is_empty() {
+        info!("Successfully downloaded private data with local address: {addr}");
+        println!("Successfully downloaded private data with local address: {addr}");
+        Ok(())
+    } else {
+        let err_no = all_errs.len();
+        eprintln!("{err_no} errors while downloading private data with local address: {addr}");
+        eprintln!("{all_errs:#?}");
+        error!("Errors while downloading private data with local address {addr}: {all_errs:#?}");
+        Err(eyre!("Errors while downloading private data"))
+    }
+}
+
+async fn download_public(
+    addr: &str,
+    address: ArchiveAddr,
+    dest_path: &str,
+    client: &mut Client,
+) -> Result<()> {
     let archive = client
         .archive_get(address)
         .await

autonomi-cli/src/commands.rs

@@ -44,10 +44,13 @@ pub enum FileCmd {
         file: String,
     },
 
-    /// Upload a file and pay for it.
+    /// Upload a file and pay for it. Data on the Network is private by default.
     Upload {
         /// The file to upload.
         file: String,
+        /// Upload the file as public. Everyone can see public data on the Network.
+        #[arg(short, long)]
+        public: bool,
     },
 
     /// Download a file from the given address.
@@ -149,7 +152,7 @@ pub async fn handle_subcommand(opt: Opt) -> Result<()> {
     match cmd {
         SubCmd::File { command } => match command {
             FileCmd::Cost { file } => file::cost(&file, peers.await?).await,
-            FileCmd::Upload { file } => file::upload(&file, peers.await?).await,
+            FileCmd::Upload { file, public } => file::upload(&file, public, peers.await?).await,
             FileCmd::Download { addr, dest_file } => {
                 file::download(&addr, &dest_file, peers.await?).await
             }

autonomi-cli/src/commands/file.rs

@@ -30,44 +30,68 @@ pub async fn cost(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
     Ok(())
 }
 
-pub async fn upload(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
+pub async fn upload(file: &str, public: bool, peers: Vec<Multiaddr>) -> Result<()> {
     let wallet = crate::keys::load_evm_wallet()?;
     let mut client = crate::actions::connect_to_network(peers).await?;
     let event_receiver = client.enable_client_events();
     let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver);
 
     println!("Uploading data to network...");
-    info!("Uploading file: {file}");
+    info!(
+        "Uploading {} file: {file}",
+        if public { "public" } else { "private" }
+    );
 
     let dir_path = PathBuf::from(file);
     let name = dir_path
         .file_name()
         .map(|n| n.to_string_lossy().to_string())
         .unwrap_or(file.to_string());
-    let xor_name = client
-        .dir_upload(dir_path, &wallet)
-        .await
-        .wrap_err("Failed to upload file")?;
-    let addr = addr_to_str(xor_name);
 
+    // upload dir
+    let local_addr;
+    let archive = if public {
+        let xor_name = client
+            .dir_upload(dir_path, &wallet)
+            .await
+            .wrap_err("Failed to upload file")?;
+        local_addr = addr_to_str(xor_name);
+        local_addr.clone()
+    } else {
+        let private_data_access = client
+            .private_dir_upload(dir_path, &wallet)
+            .await
+            .wrap_err("Failed to upload file")?;
+        local_addr = private_data_access.address();
+        private_data_access.to_hex()
+    };
+
+    // wait for upload to complete
     if let Err(e) = upload_completed_tx.send(()) {
         error!("Failed to send upload completed event: {e:?}");
         eprintln!("Failed to send upload completed event: {e:?}");
     }
 
+    // get summary
     let summary = upload_summary_thread.await?;
     if summary.record_count == 0 {
         println!("All chunks already exist on the network.");
     } else {
         println!("Successfully uploaded: {file}");
-        println!("At address: {addr}");
-        info!("Successfully uploaded: {file} at address: {addr}");
+        println!("At address: {local_addr}");
+        info!("Successfully uploaded: {file} at address: {local_addr}");
         println!("Number of chunks uploaded: {}", summary.record_count);
         println!("Total cost: {} AttoTokens", summary.tokens_spent);
     }
-    info!("Summary for upload of file {file} at {addr:?}: {summary:?}");
+    info!("Summary for upload of file {file} at {local_addr:?}: {summary:?}");
 
-    crate::user_data::write_local_file_archive(&xor_name, &name)
+    // save to local user data
+    let writer = if public {
+        crate::user_data::write_local_public_file_archive(archive, &name)
+    } else {
+        crate::user_data::write_local_private_file_archive(archive, local_addr, &name)
+    };
+    writer
         .wrap_err("Failed to save file to local user data")
         .with_suggestion(|| "Local user data saves the file address above to disk, without it you need to keep track of the address yourself")?;
     info!("Saved file to local user data");
@@ -81,11 +105,33 @@ pub async fn download(addr: &str, dest_path: &str, peers: Vec<Multiaddr>) -> Res
 }
 
 pub fn list() -> Result<()> {
+    // get public file archives
     println!("Retrieving local user data...");
-    let file_archives = crate::user_data::get_local_file_archives()?;
-    println!("✅ You have {} file archive(s):", file_archives.len());
+    let file_archives = crate::user_data::get_local_public_file_archives()
+        .wrap_err("Failed to get local public file archives")?;
+
+    println!(
+        "✅ You have {} public file archive(s):",
+        file_archives.len()
+    );
     for (addr, name) in file_archives {
         println!("{}: {}", name, addr_to_str(addr));
     }
+
+    // get private file archives
+    println!();
+    let private_file_archives = crate::user_data::get_local_private_file_archives()
+        .wrap_err("Failed to get local private file archives")?;
+
+    println!(
+        "✅ You have {} private file archive(s):",
+        private_file_archives.len()
+    );
+    for (addr, name) in private_file_archives {
+        println!("{}: {}", name, addr.address());
+    }
+
+    println!();
+    println!("> Note that private data addresses are not network addresses, they are only used for referring to private data client side.");
     Ok(())
 }