From 8616c7a2d4d83579dd2dba5f81032c9ac529bb90 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Oct 2024 12:30:15 +0900 Subject: [PATCH 1/2] feat: private data, private archives, vault support and CLI integration --- Cargo.lock | 6 +- autonomi-cli/Cargo.toml | 2 + autonomi-cli/src/access/user_data.rs | 77 +++++++++++++- autonomi-cli/src/actions/download.rs | 78 +++++++++++++- autonomi-cli/src/commands.rs | 7 +- autonomi-cli/src/commands/file.rs | 72 ++++++++++--- autonomi-cli/src/commands/vault.rs | 20 +++- autonomi/src/client/archive.rs | 1 + autonomi/src/client/archive_private.rs | 140 +++++++++++++++++++++++++ autonomi/src/client/data.rs | 44 ++++---- autonomi/src/client/data_private.rs | 138 ++++++++++++++++++++++++ autonomi/src/client/fs.rs | 2 +- autonomi/src/client/fs_private.rs | 101 ++++++++++++++++++ autonomi/src/client/mod.rs | 6 ++ autonomi/src/client/vault/user_data.rs | 3 + evmlib/src/contract/network_token.rs | 2 +- 16 files changed, 649 insertions(+), 50 deletions(-) create mode 100644 autonomi/src/client/archive_private.rs create mode 100644 autonomi/src/client/data_private.rs create mode 100644 autonomi/src/client/fs_private.rs diff --git a/Cargo.lock b/Cargo.lock index ef840ca3a9..d274255dbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1134,6 +1134,8 @@ dependencies = [ "indicatif", "rand 0.8.5", "rayon", + "serde", + "serde_json", "sn_build_info", "sn_logging", "sn_peers_acquisition", @@ -7813,9 +7815,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", diff --git a/autonomi-cli/Cargo.toml b/autonomi-cli/Cargo.toml index 775d5f7f86..7e71a4a841 100644 --- a/autonomi-cli/Cargo.toml +++ b/autonomi-cli/Cargo.toml @@ -49,6 +49,8 @@ sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.5.4" } sn_build_info = { path = "../sn_build_info", version = "0.1.16" } sn_logging = { path = "../sn_logging", version = "0.2.37" } walkdir = "2.5.0" +serde_json = "1.0.132" +serde = "1.0.210" [dev-dependencies] autonomi = { path = "../autonomi", version = "0.2.1", features = [ diff --git a/autonomi-cli/src/access/user_data.rs b/autonomi-cli/src/access/user_data.rs index 799c23c0d7..57deb85785 100644 --- a/autonomi-cli/src/access/user_data.rs +++ b/autonomi-cli/src/access/user_data.rs @@ -11,6 +11,7 @@ use std::collections::HashMap; use autonomi::client::{ address::{addr_to_str, str_to_addr}, archive::ArchiveAddr, + archive_private::PrivateArchiveAccess, registers::{RegisterAddress, RegisterSecretKey}, vault::UserData, }; @@ -21,19 +22,62 @@ use super::{ keys::{create_register_signing_key_file, get_register_signing_key}, }; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +struct PrivateFileArchive { + name: String, + secret_access: String, +} + pub fn get_local_user_data() -> Result { let register_sk = get_register_signing_key().map(|k| k.to_hex()).ok(); let registers = get_local_registers()?; - let file_archives = get_local_file_archives()?; + let file_archives = get_local_public_file_archives()?; + let private_file_archives = get_local_private_file_archives()?; let user_data = UserData { register_sk, registers, file_archives, + private_file_archives, }; Ok(user_data) } +pub fn get_local_private_file_archives() -> Result> { + let data_dir = get_client_data_dir_path()?; + let user_data_path = data_dir.join("user_data"); + let private_file_archives_path = user_data_path.join("private_file_archives"); + std::fs::create_dir_all(&private_file_archives_path)?; + + let mut private_file_archives = HashMap::new(); + for entry in walkdir::WalkDir::new(private_file_archives_path) + .min_depth(1) + .max_depth(1) + { + let entry = entry?; + let file_content = std::fs::read_to_string(entry.path())?; + let private_file_archive: PrivateFileArchive = serde_json::from_str(&file_content)?; + let private_file_archive_access = + PrivateArchiveAccess::from_hex(&private_file_archive.secret_access)?; + private_file_archives.insert(private_file_archive_access, private_file_archive.name); + } + Ok(private_file_archives) +} + +pub fn get_local_private_archive_access(local_addr: &str) -> Result { + let data_dir = get_client_data_dir_path()?; + let user_data_path = data_dir.join("user_data"); + let private_file_archives_path = user_data_path.join("private_file_archives"); + let file_path = private_file_archives_path.join(local_addr); + let file_content = std::fs::read_to_string(file_path)?; + let private_file_archive: PrivateFileArchive = serde_json::from_str(&file_content)?; + let private_file_archive_access = + PrivateArchiveAccess::from_hex(&private_file_archive.secret_access)?; + Ok(private_file_archive_access) +} + pub fn get_local_registers() -> Result> { let data_dir = get_client_data_dir_path()?; let user_data_path = data_dir.join("user_data"); @@ -55,7 +99,7 @@ pub fn get_local_registers() -> Result> { Ok(registers) } -pub fn get_local_file_archives() -> Result> { +pub fn get_local_public_file_archives() -> Result> { let data_dir = get_client_data_dir_path()?; let user_data_path = data_dir.join("user_data"); let file_archives_path = user_data_path.join("file_archives"); @@ -86,8 +130,13 @@ pub fn write_local_user_data(user_data: &UserData) -> Result<()> { } for (archive, name) in user_data.file_archives.iter() { - write_local_file_archive(archive, name)?; + write_local_public_file_archive(addr_to_str(*archive), name)?; + } + + for (archive, name) in user_data.private_file_archives.iter() { + write_local_private_file_archive(archive.to_hex(), archive.address(), name)?; } + Ok(()) } @@ -100,11 +149,29 @@ pub fn write_local_register(register: &RegisterAddress, name: &str) -> Result<() Ok(()) } -pub fn write_local_file_archive(archive: &ArchiveAddr, name: &str) -> Result<()> { +pub fn write_local_public_file_archive(archive: String, name: &str) -> Result<()> { let data_dir = get_client_data_dir_path()?; let user_data_path = data_dir.join("user_data"); let file_archives_path = user_data_path.join("file_archives"); std::fs::create_dir_all(&file_archives_path)?; - std::fs::write(file_archives_path.join(addr_to_str(*archive)), name)?; + std::fs::write(file_archives_path.join(archive), name)?; + Ok(()) +} + +pub fn write_local_private_file_archive( + archive: String, + local_addr: String, + name: &str, +) -> Result<()> { + let data_dir = get_client_data_dir_path()?; + let user_data_path = data_dir.join("user_data"); + let private_file_archives_path = user_data_path.join("private_file_archives"); + std::fs::create_dir_all(&private_file_archives_path)?; + let file_name = local_addr; + let content = serde_json::to_string(&PrivateFileArchive { + name: name.to_string(), + secret_access: archive, + })?; + std::fs::write(private_file_archives_path.join(file_name), content)?; Ok(()) } diff --git a/autonomi-cli/src/actions/download.rs b/autonomi-cli/src/actions/download.rs index 069a37c0eb..7beb3578f1 100644 --- a/autonomi-cli/src/actions/download.rs +++ b/autonomi-cli/src/actions/download.rs @@ -7,12 +7,84 @@ // permissions and limitations relating to use of the SAFE Network Software. use super::get_progress_bar; -use autonomi::{client::address::str_to_addr, Client}; -use color_eyre::eyre::{eyre, Context, Result}; +use autonomi::{ + client::{address::str_to_addr, archive::ArchiveAddr, archive_private::PrivateArchiveAccess}, + Client, +}; +use color_eyre::{ + eyre::{eyre, Context, Result}, + Section, +}; use std::path::PathBuf; pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Result<()> { - let address = str_to_addr(addr).wrap_err("Failed to parse data address")?; + let public_address = str_to_addr(addr).ok(); + let private_address = crate::user_data::get_local_private_archive_access(addr) + .inspect_err(|e| error!("Failed to get private archive access: {e}")) + .ok(); + + match (public_address, private_address) { + (Some(public_address), _) => download_public(addr, public_address, dest_path, client).await, + (_, Some(private_address)) => download_private(addr, private_address, dest_path, client).await, + _ => Err(eyre!("Failed to parse data address")) + .with_suggestion(|| "Public addresses look like this: 0037cfa13eae4393841cbc00c3a33cade0f98b8c1f20826e5c51f8269e7b09d7") + .with_suggestion(|| "Private addresses look like this: 1358645341480028172") + .with_suggestion(|| "Try the `file list` command to get addresses you have access to"), + } +} + +async fn download_private( + addr: &str, + private_address: PrivateArchiveAccess, + dest_path: &str, + client: &mut Client, +) -> Result<()> { + let archive = client + .private_archive_get(private_address) + .await + .wrap_err("Failed to fetch data from address")?; + + let progress_bar = get_progress_bar(archive.iter().count() as u64)?; + let mut all_errs = vec![]; + for (path, access, _meta) in archive.iter() { + progress_bar.println(format!("Fetching file: {path:?}...")); + let bytes = match client.private_data_get(access.clone()).await { + Ok(bytes) => bytes, + Err(e) => { + let err = format!("Failed to fetch file {path:?}: {e}"); + all_errs.push(err); + continue; + } + }; + + let path = PathBuf::from(dest_path).join(path); + let here = PathBuf::from("."); + let parent = path.parent().unwrap_or_else(|| &here); + std::fs::create_dir_all(parent)?; + std::fs::write(path, bytes)?; + progress_bar.clone().inc(1); + } + progress_bar.finish_and_clear(); + + if all_errs.is_empty() { + info!("Successfully downloaded private data with local address: {addr}"); + println!("Successfully downloaded private data with local address: {addr}"); + Ok(()) + } else { + let err_no = all_errs.len(); + eprintln!("{err_no} errors while downloading private data with local address: {addr}"); + eprintln!("{all_errs:#?}"); + error!("Errors while downloading private data with local address {addr}: {all_errs:#?}"); + Err(eyre!("Errors while downloading private data")) + } +} + +async fn download_public( + addr: &str, + address: ArchiveAddr, + dest_path: &str, + client: &mut Client, +) -> Result<()> { let archive = client .archive_get(address) .await diff --git a/autonomi-cli/src/commands.rs b/autonomi-cli/src/commands.rs index 06adb34006..c374eca78f 100644 --- a/autonomi-cli/src/commands.rs +++ b/autonomi-cli/src/commands.rs @@ -44,10 +44,13 @@ pub enum FileCmd { file: String, }, - /// Upload a file and pay for it. + /// Upload a file and pay for it. Data on the Network is private by default. Upload { /// The file to upload. file: String, + /// Upload the file as public. Everyone can see public data on the Network. + #[arg(short, long)] + public: bool, }, /// Download a file from the given address. @@ -149,7 +152,7 @@ pub async fn handle_subcommand(opt: Opt) -> Result<()> { match cmd { SubCmd::File { command } => match command { FileCmd::Cost { file } => file::cost(&file, peers.await?).await, - FileCmd::Upload { file } => file::upload(&file, peers.await?).await, + FileCmd::Upload { file, public } => file::upload(&file, public, peers.await?).await, FileCmd::Download { addr, dest_file } => { file::download(&addr, &dest_file, peers.await?).await } diff --git a/autonomi-cli/src/commands/file.rs b/autonomi-cli/src/commands/file.rs index faf21137e6..e32b98b51d 100644 --- a/autonomi-cli/src/commands/file.rs +++ b/autonomi-cli/src/commands/file.rs @@ -30,44 +30,68 @@ pub async fn cost(file: &str, peers: Vec) -> Result<()> { Ok(()) } -pub async fn upload(file: &str, peers: Vec) -> Result<()> { +pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<()> { let wallet = crate::keys::load_evm_wallet()?; let mut client = crate::actions::connect_to_network(peers).await?; let event_receiver = client.enable_client_events(); let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver); println!("Uploading data to network..."); - info!("Uploading file: {file}"); + info!( + "Uploading {} file: {file}", + if public { "public" } else { "private" } + ); let dir_path = PathBuf::from(file); let name = dir_path .file_name() .map(|n| n.to_string_lossy().to_string()) .unwrap_or(file.to_string()); - let xor_name = client - .dir_upload(dir_path, &wallet) - .await - .wrap_err("Failed to upload file")?; - let addr = addr_to_str(xor_name); + // upload dir + let local_addr; + let archive = if public { + let xor_name = client + .dir_upload(dir_path, &wallet) + .await + .wrap_err("Failed to upload file")?; + local_addr = addr_to_str(xor_name); + local_addr.clone() + } else { + let private_data_access = client + .private_dir_upload(dir_path, &wallet) + .await + .wrap_err("Failed to upload file")?; + local_addr = private_data_access.address(); + private_data_access.to_hex() + }; + + // wait for upload to complete if let Err(e) = upload_completed_tx.send(()) { error!("Failed to send upload completed event: {e:?}"); eprintln!("Failed to send upload completed event: {e:?}"); } + // get summary let summary = upload_summary_thread.await?; if summary.record_count == 0 { println!("All chunks already exist on the network."); } else { println!("Successfully uploaded: {file}"); - println!("At address: {addr}"); - info!("Successfully uploaded: {file} at address: {addr}"); + println!("At address: {local_addr}"); + info!("Successfully uploaded: {file} at address: {local_addr}"); println!("Number of chunks uploaded: {}", summary.record_count); println!("Total cost: {} AttoTokens", summary.tokens_spent); } - info!("Summary for upload of file {file} at {addr:?}: {summary:?}"); + info!("Summary for upload of file {file} at {local_addr:?}: {summary:?}"); - crate::user_data::write_local_file_archive(&xor_name, &name) + // save to local user data + let writer = if public { + crate::user_data::write_local_public_file_archive(archive, &name) + } else { + crate::user_data::write_local_private_file_archive(archive, local_addr, &name) + }; + writer .wrap_err("Failed to save file to local user data") .with_suggestion(|| "Local user data saves the file address above to disk, without it you need to keep track of the address yourself")?; info!("Saved file to local user data"); @@ -81,11 +105,33 @@ pub async fn download(addr: &str, dest_path: &str, peers: Vec) -> Res } pub fn list() -> Result<()> { + // get public file archives println!("Retrieving local user data..."); - let file_archives = crate::user_data::get_local_file_archives()?; - println!("✅ You have {} file archive(s):", file_archives.len()); + let file_archives = crate::user_data::get_local_public_file_archives() + .wrap_err("Failed to get local public file archives")?; + + println!( + "✅ You have {} public file archive(s):", + file_archives.len() + ); for (addr, name) in file_archives { println!("{}: {}", name, addr_to_str(addr)); } + + // get private file archives + println!(); + let private_file_archives = crate::user_data::get_local_private_file_archives() + .wrap_err("Failed to get local private file archives")?; + + println!( + "✅ You have {} private file archive(s):", + private_file_archives.len() + ); + for (addr, name) in private_file_archives { + println!("{}: {}", name, addr.address()); + } + + println!(); + println!("> Note that private data addresses are not network addresses, they are only used for referring to private data client side."); Ok(()) } diff --git a/autonomi-cli/src/commands/vault.rs b/autonomi-cli/src/commands/vault.rs index 9888366eec..60c0c8192f 100644 --- a/autonomi-cli/src/commands/vault.rs +++ b/autonomi-cli/src/commands/vault.rs @@ -34,6 +34,7 @@ pub async fn create(peers: Vec) -> Result<()> { println!("Retrieving local user data..."); let local_user_data = crate::user_data::get_local_user_data()?; let file_archives_len = local_user_data.file_archives.len(); + let private_file_archives_len = local_user_data.private_file_archives.len(); let registers_len = local_user_data.registers.len(); println!("Pushing to network vault..."); @@ -48,7 +49,10 @@ pub async fn create(peers: Vec) -> Result<()> { } println!("Total cost: {total_cost} AttoTokens"); - println!("Vault contains {file_archives_len} file archive(s) and {registers_len} register(s)"); + println!("Vault contains:"); + println!("{file_archives_len} public file archive(s)"); + println!("{private_file_archives_len} private file archive(s)"); + println!("{registers_len} register(s)"); Ok(()) } @@ -74,13 +78,17 @@ pub async fn sync(peers: Vec, force: bool) -> Result<()> { println!("Pushing local user data to network vault..."); let local_user_data = crate::user_data::get_local_user_data()?; let file_archives_len = local_user_data.file_archives.len(); + let private_file_archives_len = local_user_data.private_file_archives.len(); let registers_len = local_user_data.registers.len(); client .put_user_data_to_vault(&vault_sk, &wallet, local_user_data) .await?; println!("✅ Successfully synced vault"); - println!("Vault contains {file_archives_len} file archive(s) and {registers_len} register(s)"); + println!("Vault contains:"); + println!("{file_archives_len} public file archive(s)"); + println!("{private_file_archives_len} private file archive(s)"); + println!("{registers_len} register(s)"); Ok(()) } @@ -93,10 +101,12 @@ pub async fn load(peers: Vec) -> Result<()> { println!("Writing user data to disk..."); crate::user_data::write_local_user_data(&user_data)?; + println!("✅ Successfully loaded vault with:"); + println!("{} public file archive(s)", user_data.file_archives.len()); println!( - "✅ Successfully loaded vault with {} file archive(s) and {} register(s)", - user_data.file_archives.len(), - user_data.registers.len() + "{} private file archive(s)", + user_data.private_file_archives.len() ); + println!("{} register(s)", user_data.registers.len()); Ok(()) } diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index 3957b3d942..04ad120b19 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -35,6 +35,7 @@ pub enum RenameError { /// An archive of files that containing file paths, their metadata and the files data addresses /// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. +/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`]. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct Archive { map: HashMap, diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/archive_private.rs new file mode 100644 index 0000000000..a7ba854380 --- /dev/null +++ b/autonomi/src/client/archive_private.rs @@ -0,0 +1,140 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use sn_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; + +use super::{ + archive::{Metadata, RenameError}, + data::{GetError, PutError}, + data_private::PrivateDataAccess, + Client, +}; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use sn_evm::EvmWallet; + +/// The address of a private archive +/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data +pub type PrivateArchiveAccess = PrivateDataAccess; + +/// A private archive of files that containing file paths, their metadata and the files data maps +/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct PrivateArchive { + map: HashMap, +} + +impl PrivateArchive { + /// Create a new emtpy local archive + /// Note that this does not upload the archive to the network + pub fn new() -> Self { + Self { + map: HashMap::new(), + } + } + + /// Rename a file in an archive + /// Note that this does not upload the archive to the network + pub fn rename_file(&mut self, old_path: &Path, new_path: &Path) -> Result<(), RenameError> { + let (data_addr, mut meta) = self + .map + .remove(old_path) + .ok_or(RenameError::FileNotFound(old_path.to_path_buf()))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + meta.modified = now; + self.map.insert(new_path.to_path_buf(), (data_addr, meta)); + Ok(()) + } + + /// Add a file to a local archive + /// Note that this does not upload the archive to the network + pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) { + self.map.insert(path, (data_map, meta)); + } + + /// Add a file to a local archive, with default metadata + /// Note that this does not upload the archive to the network + pub fn add_new_file(&mut self, path: PathBuf, data_map: PrivateDataAccess) { + self.map.insert(path, (data_map, Metadata::new())); + } + + /// List all files in the archive + pub fn files(&self) -> Vec<(PathBuf, Metadata)> { + self.map + .iter() + .map(|(path, (_, meta))| (path.clone(), meta.clone())) + .collect() + } + + /// List all data addresses of the files in the archive + pub fn addresses(&self) -> Vec { + self.map + .values() + .map(|(data_map, _)| data_map.clone()) + .collect() + } + + /// Iterate over the archive items + /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) + pub fn iter(&self) -> impl Iterator { + self.map + .iter() + .map(|(path, (data_map, meta))| (path, data_map, meta)) + } + + /// Get the underlying map + pub fn map(&self) -> &HashMap { + &self.map + } + + /// Deserialize from bytes. + pub fn from_bytes(data: Bytes) -> Result { + let root: PrivateArchive = rmp_serde::from_slice(&data[..])?; + + Ok(root) + } + + /// Serialize to bytes. + pub fn into_bytes(&self) -> Result { + let root_serialized = rmp_serde::to_vec(&self)?; + let root_serialized = Bytes::from(root_serialized); + + Ok(root_serialized) + } +} + +impl Client { + /// Fetch a private archive from the network + pub async fn private_archive_get( + &self, + addr: PrivateArchiveAccess, + ) -> Result { + let data = self.private_data_get(addr).await?; + Ok(PrivateArchive::from_bytes(data)?) + } + + /// Upload a private archive to the network + pub async fn private_archive_put( + &self, + archive: PrivateArchive, + wallet: &EvmWallet, + ) -> Result { + let bytes = archive + .into_bytes() + .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; + self.private_data_put(bytes, wallet).await + } +} diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data.rs index 6fda246380..869022cd37 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data.rs @@ -8,7 +8,7 @@ use bytes::Bytes; use libp2p::kad::Quorum; -use tokio::task::JoinError; +use tokio::task::{JoinError, JoinSet}; use std::collections::HashSet; use xor_name::XorName; @@ -47,6 +47,8 @@ pub enum PutError { VaultBadOwner, #[error("Payment unexpectedly invalid for {0:?}")] PaymentUnexpectedlyInvalid(NetworkAddress), + #[error("Could not simultaneously upload chunks: {0:?}")] + JoinError(tokio::task::JoinError), } /// Errors that can occur during the pay operation. @@ -102,8 +104,9 @@ impl Client { Ok(data) } - /// Upload a piece of data to the network. This data will be self-encrypted. + /// Upload a piece of data to the network. /// Returns the Data Address at which the data was stored. + /// This data is publicly accessible. pub async fn data_put(&self, data: Bytes, wallet: &EvmWallet) -> Result { let now = sn_networking::target_arch::Instant::now(); let (data_map_chunk, chunks) = encrypt(data)?; @@ -130,26 +133,31 @@ impl Client { let mut record_count = 0; - // Upload data map - if let Some(proof) = payment_proofs.get(&map_xor_name) { - debug!("Uploading data map chunk: {map_xor_name:?}"); - self.chunk_upload_with_payment(data_map_chunk.clone(), proof.clone()) - .await - .inspect_err(|err| error!("Error uploading data map chunk: {err:?}"))?; - record_count += 1; - } - - // Upload the rest of the chunks + // Upload all the chunks in parallel including the data map chunk debug!("Uploading {} chunks", chunks.len()); - for chunk in chunks { + let mut tasks = JoinSet::new(); + for chunk in chunks.into_iter().chain(std::iter::once(data_map_chunk)) { + let self_clone = self.clone(); + let address = *chunk.address(); if let Some(proof) = payment_proofs.get(chunk.name()) { - let address = *chunk.address(); - self.chunk_upload_with_payment(chunk, proof.clone()) - .await - .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}"))?; - record_count += 1; + let proof_clone = proof.clone(); + tasks.spawn(async move { + self_clone + .chunk_upload_with_payment(chunk, proof_clone) + .await + .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}")) + }); + } else { + debug!("Chunk at {address:?} was already paid for so skipping"); } } + while let Some(result) = tasks.join_next().await { + result + .inspect_err(|err| error!("Join error uploading chunk: {err:?}")) + .map_err(PutError::JoinError)? + .inspect_err(|err| error!("Error uploading chunk: {err:?}"))?; + record_count += 1; + } if let Some(channel) = self.client_event_sender.as_ref() { let tokens_spent = payment_proofs diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data_private.rs new file mode 100644 index 0000000000..b6d0bfa8a3 --- /dev/null +++ b/autonomi/src/client/data_private.rs @@ -0,0 +1,138 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::hash::{DefaultHasher, Hash, Hasher}; + +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use sn_evm::{Amount, EvmWallet}; +use sn_protocol::storage::Chunk; +use tokio::task::JoinSet; + +use super::data::{GetError, PutError}; +use crate::client::{ClientEvent, UploadSummary}; +use crate::{self_encryption::encrypt, Client}; + +/// Private data on the network can be accessed with this +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PrivateDataAccess(Chunk); + +impl PrivateDataAccess { + pub fn to_hex(&self) -> String { + hex::encode(self.0.value()) + } + + pub fn from_hex(hex: &str) -> Result { + let data = hex::decode(hex)?; + Ok(Self(Chunk::new(Bytes::from(data)))) + } + + /// Get a private address for [`PrivateDataAccess`]. Note that this is not a network address, it is only used for refering to private data client side. + pub fn address(&self) -> String { + hash_to_short_string(&self.to_hex()) + } +} + +fn hash_to_short_string(input: &str) -> String { + let mut hasher = DefaultHasher::new(); + input.hash(&mut hasher); + let hash_value = hasher.finish(); + hash_value.to_string() +} + +impl Client { + /// Fetch a blob of private data from the network + pub async fn private_data_get(&self, data_map: PrivateDataAccess) -> Result { + info!( + "Fetching private data from Data Map {:?}", + data_map.0.address() + ); + let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; + + Ok(data) + } + + /// Upload a piece of private data to the network. This data will be self-encrypted. + /// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks. + /// This data is private and only accessible with the [`PrivateDataAccess`]. + pub async fn private_data_put( + &self, + data: Bytes, + wallet: &EvmWallet, + ) -> Result { + let now = sn_networking::target_arch::Instant::now(); + let (data_map_chunk, chunks) = encrypt(data)?; + debug!("Encryption took: {:.2?}", now.elapsed()); + + // Pay for all chunks + let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); + info!("Paying for {} addresses", xor_names.len()); + let (payment_proofs, _free_chunks) = self + .pay(xor_names.into_iter(), wallet) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}"))?; + + // Upload the chunks with the payments + let mut record_count = 0; + debug!("Uploading {} chunks", chunks.len()); + let mut tasks = JoinSet::new(); + for chunk in chunks { + let self_clone = self.clone(); + let address = *chunk.address(); + if let Some(proof) = payment_proofs.get(chunk.name()) { + let proof_clone = proof.clone(); + tasks.spawn(async move { + self_clone + .chunk_upload_with_payment(chunk, proof_clone) + .await + .inspect_err(|err| error!("Error uploading chunk {address:?} :{err:?}")) + }); + } else { + debug!("Chunk at {address:?} was already paid for so skipping"); + } + } + while let Some(result) = tasks.join_next().await { + result + .inspect_err(|err| error!("Join error uploading chunk: {err:?}")) + .map_err(PutError::JoinError)? + .inspect_err(|err| error!("Error uploading chunk: {err:?}"))?; + record_count += 1; + } + + // Reporting + if let Some(channel) = self.client_event_sender.as_ref() { + let tokens_spent = payment_proofs + .values() + .map(|proof| proof.quote.cost.as_atto()) + .sum::(); + + let summary = UploadSummary { + record_count, + tokens_spent, + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err:?}"); + } + } + + Ok(PrivateDataAccess(data_map_chunk)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hex() { + let data_map = PrivateDataAccess(Chunk::new(Bytes::from_static(b"hello"))); + let hex = data_map.to_hex(); + let data_map2 = PrivateDataAccess::from_hex(&hex).expect("Failed to decode hex"); + assert_eq!(data_map, data_map2); + } +} diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/fs.rs index 51311e2f70..d7f243df68 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/fs.rs @@ -179,7 +179,7 @@ impl Client { // Get metadata from directory entry. Defaults to `0` for creation and modification times if // any error is encountered. Logs errors upon error. -fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { +pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { let fs_metadata = match entry.metadata() { Ok(metadata) => metadata, Err(err) => { diff --git a/autonomi/src/client/fs_private.rs b/autonomi/src/client/fs_private.rs new file mode 100644 index 0000000000..0d9b819d70 --- /dev/null +++ b/autonomi/src/client/fs_private.rs @@ -0,0 +1,101 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::client::Client; +use bytes::Bytes; +use sn_evm::EvmWallet; +use std::path::PathBuf; + +use super::archive_private::{PrivateArchive, PrivateArchiveAccess}; +use super::data_private::PrivateDataAccess; +use super::fs::{DownloadError, UploadError}; + +impl Client { + /// Download a private file from network to local file system + pub async fn private_file_download( + &self, + data_access: PrivateDataAccess, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let data = self.private_data_get(data_access).await?; + if let Some(parent) = to_dest.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(to_dest, data).await?; + Ok(()) + } + + /// Download a private directory from network to local file system + pub async fn private_dir_download( + &self, + archive_access: PrivateArchiveAccess, + to_dest: PathBuf, + ) -> Result<(), DownloadError> { + let archive = self.private_archive_get(archive_access).await?; + for (path, addr, _meta) in archive.iter() { + self.private_file_download(addr.clone(), to_dest.join(path)) + .await?; + } + Ok(()) + } + + /// Upload a private directory to the network. The directory is recursively walked. + /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) + pub async fn private_dir_upload( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let mut archive = PrivateArchive::new(); + + for entry in walkdir::WalkDir::new(dir_path) { + let entry = entry?; + + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path().to_path_buf(); + tracing::info!("Uploading file: {path:?}"); + #[cfg(feature = "loud")] + println!("Uploading file: {path:?}"); + let file = self.private_file_upload(path.clone(), wallet).await?; + + let metadata = super::fs::metadata_from_entry(&entry); + + archive.add_file(path, file, metadata); + } + + let archive_serialized = archive.into_bytes()?; + + let arch_addr = self.private_data_put(archive_serialized, wallet).await?; + + Ok(arch_addr) + } + + /// Upload a private file to the network. + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) + async fn private_file_upload( + &self, + path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let data = tokio::fs::read(path).await?; + let data = Bytes::from(data); + let addr = self.private_data_put(data, wallet).await?; + Ok(addr) + } +} diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 4771d19e2a..d530f210f2 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -11,11 +11,17 @@ pub mod address; #[cfg(feature = "data")] pub mod archive; #[cfg(feature = "data")] +pub mod archive_private; +#[cfg(feature = "data")] pub mod data; +#[cfg(feature = "data")] +pub mod data_private; #[cfg(feature = "external-signer")] pub mod external_signer; #[cfg(feature = "fs")] pub mod fs; +#[cfg(feature = "fs")] +pub mod fs_private; #[cfg(feature = "registers")] pub mod registers; #[cfg(feature = "vault")] diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/vault/user_data.rs index 736bd6292d..1f91b547bb 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/vault/user_data.rs @@ -9,6 +9,7 @@ use std::collections::HashMap; use crate::client::archive::ArchiveAddr; +use crate::client::archive_private::PrivateArchiveAccess; use crate::client::data::GetError; use crate::client::data::PutError; use crate::client::registers::RegisterAddress; @@ -37,6 +38,8 @@ pub struct UserData { pub registers: HashMap, /// Owned file archive addresses, along with their names (can be empty) pub file_archives: HashMap, + /// Owned private file archives, along with their names (can be empty) + pub private_file_archives: HashMap, } /// Errors that can occur during the get operation. diff --git a/evmlib/src/contract/network_token.rs b/evmlib/src/contract/network_token.rs index 013d572037..10903c9fd2 100644 --- a/evmlib/src/contract/network_token.rs +++ b/evmlib/src/contract/network_token.rs @@ -52,7 +52,7 @@ where pub async fn deploy(provider: P) -> Self { let contract = NetworkTokenContract::deploy(provider) .await - .expect("Could not deploy contract"); + .expect("Could not deploy contract, update anvil by running `foundryup` and try again"); NetworkToken { contract } } From d8cad7c4daf93ebc85dc4276e5f1352e356e9267 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Oct 2024 12:46:04 +0900 Subject: [PATCH 2/2] ci: fix upload download from different clients --- .github/workflows/memcheck.yml | 2 +- autonomi-cli/src/actions/download.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index c2e5406207..cbfb52d4cc 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -70,7 +70,7 @@ jobs: shell: bash - name: File upload - run: ./target/release/autonomi --log-output-dest=data-dir file upload "./the-test-data.zip" > ./upload_output 2>&1 + run: ./target/release/autonomi --log-output-dest=data-dir file upload --public "./the-test-data.zip" > ./upload_output 2>&1 env: SN_LOG: "v" timeout-minutes: 5 diff --git a/autonomi-cli/src/actions/download.rs b/autonomi-cli/src/actions/download.rs index 7beb3578f1..ff737ac2c1 100644 --- a/autonomi-cli/src/actions/download.rs +++ b/autonomi-cli/src/actions/download.rs @@ -26,7 +26,7 @@ pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Resul match (public_address, private_address) { (Some(public_address), _) => download_public(addr, public_address, dest_path, client).await, (_, Some(private_address)) => download_private(addr, private_address, dest_path, client).await, - _ => Err(eyre!("Failed to parse data address")) + _ => Err(eyre!("Failed to parse data address {addr}")) .with_suggestion(|| "Public addresses look like this: 0037cfa13eae4393841cbc00c3a33cade0f98b8c1f20826e5c51f8269e7b09d7") .with_suggestion(|| "Private addresses look like this: 1358645341480028172") .with_suggestion(|| "Try the `file list` command to get addresses you have access to"),