diff --git a/api/src/config.rs b/api/src/config.rs index 315cecb8dd8..480e8abcbf4 100644 --- a/api/src/config.rs +++ b/api/src/config.rs @@ -1878,6 +1878,16 @@ mod tests { assert_eq!(&config.id, ""); } + #[test] + fn test_backend_http_proxy_config() { + let config = + r#"{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"/tmp"}}}"#; + let config = ConfigV2::from_str(config).unwrap(); + let backend = config.backend.unwrap(); + assert_eq!(&backend.backend_type, "http-proxy"); + assert_eq!(&backend.http_proxy.unwrap().addr, "/tmp"); + } + #[test] fn test_new_localfs() { let config = ConfigV2::new_localfs("id1", "./").unwrap(); diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index fc88bde7e0f..10917e26d83 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -24,9 +24,11 @@ use clap::parser::ValueSource; use clap::{Arg, ArgAction, ArgMatches, Command as App}; use nix::unistd::{getegid, geteuid}; use nydus::get_build_time_info; -use nydus_api::{BuildTimeInfo, ConfigV2}; +use nydus_api::{BuildTimeInfo, ConfigV2, LocalFsConfig}; use nydus_app::setup_logging; use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion}; +use nydus_storage::backend::localfs::LocalFs; +use nydus_storage::backend::BlobBackend; use nydus_storage::device::BlobFeatures; use nydus_storage::factory::BlobFactory; use nydus_storage::meta::format_blob_features; @@ -525,6 +527,12 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .help("File path of RAFS metadata") .required_unless_present("bootstrap"), ) + .arg( + Arg::new("backend-config") + .long("backend-config") + .help("config file of backend") + .required(false), + ) .arg( Arg::new("bootstrap") .short('B') @@ -955,10 +963,7 @@ impl Command { Some(args) => Some(import_chunk_dict(args, config, &rs.meta.get_config())?), }; - let cfg_file = matches.get_one::("backend-config").unwrap(); - let cfg = ConfigV2::from_file(cfg_file)?; - let backend_cfg = cfg.get_backend_config()?; - let backend = BlobFactory::new_backend(backend_cfg, "compactor")?; + let backend = Self::get_backend(matches, "compactor")?; let config_file_path = matches.get_one::("config").unwrap(); let file = File::open(config_file_path) @@ -984,9 +989,25 @@ impl Command { if output.is_empty() { return Err(anyhow!("invalid empty --output option")); } + let blob = matches.get_one::("blob").map(|s| s.as_str()); + let backend: Option> = match blob { + Some(blob_path) => { + let blob_path = PathBuf::from(blob_path); + let local_fs_conf = LocalFsConfig { + blob_file: blob_path.to_str().unwrap().to_owned(), + dir: Default::default(), + alt_dirs: Default::default(), + }; + let local_fs = LocalFs::new(&local_fs_conf, Some("unpacker")) + .with_context(|| format!("fail to create local backend for {:?}", blob_path))?; + + Some(Arc::new(local_fs)) + } + None => Self::get_backend(matches, "unpacker").ok(), + }; - OCIUnpacker::new(bootstrap, blob, output) + OCIUnpacker::new(bootstrap, backend, output) .with_context(|| "fail to create unpacker")? .unpack(config) .with_context(|| "fail to unpack") @@ -1210,6 +1231,20 @@ impl Command { Ok(Arc::new(config)) } + fn get_backend( + matches: &ArgMatches, + blob_id: &str, + ) -> Result> { + let cfg_file = matches + .get_one::("backend-config") + .context("missing backend-config argument")?; + let cfg = ConfigV2::from_file(cfg_file)?; + let backend_cfg = cfg.get_backend_config()?; + let backend = BlobFactory::new_backend(backend_cfg, blob_id)?; + + Ok(backend) + } + fn get_blob_id(matches: &ArgMatches) -> Result { let mut blob_id = String::new(); diff --git a/src/bin/nydus-image/unpack/mod.rs b/src/bin/nydus-image/unpack/mod.rs index 643795db8d6..184b771ecbe 100644 --- a/src/bin/nydus-image/unpack/mod.rs +++ b/src/bin/nydus-image/unpack/mod.rs @@ -10,15 +10,13 @@ use std::str; use std::sync::Arc; use anyhow::{Context, Result}; -use nydus_api::{ConfigV2, LocalFsConfig}; +use nydus_api::ConfigV2; use nydus_rafs::{ metadata::{RafsInodeExt, RafsSuper}, RafsIterator, }; -use nydus_storage::{ - backend::{localfs::LocalFs, BlobBackend, BlobReader}, - device::BlobInfo, -}; +use nydus_storage::backend::BlobBackend; +use nydus_storage::device::BlobInfo; use tar::{Builder, Header}; use self::pax::{ @@ -36,24 +34,27 @@ pub trait Unpacker { /// A unpacker with the ability to convert bootstrap file and blob file to tar pub struct OCIUnpacker { bootstrap: PathBuf, - blob: Option, + blob_backend: Option>, output: PathBuf, builder_factory: OCITarBuilderFactory, } impl OCIUnpacker { - pub fn new(bootstrap: &Path, blob: Option<&str>, output: &str) -> Result { + pub fn new( + bootstrap: &Path, + blob_backend: Option>, + output: &str, + ) -> Result { let bootstrap = bootstrap.to_path_buf(); let output = PathBuf::from(output); - let blob = blob.map(PathBuf::from); let builder_factory = OCITarBuilderFactory::new(); Ok(OCIUnpacker { builder_factory, bootstrap, - blob, + blob_backend, output, }) } @@ -72,15 +73,15 @@ impl OCIUnpacker { impl Unpacker for OCIUnpacker { fn unpack(&self, config: Arc) -> Result<()> { debug!( - "oci unpacker, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", - self.bootstrap, self.blob, self.output + "oci unpacker, bootstrap file: {:?}, output file: {:?}", + self.bootstrap, self.output ); let rafs = self.load_rafs(config)?; let mut builder = self .builder_factory - .create(&rafs, self.blob.as_deref(), &self.output)?; + .create(&rafs, &self.blob_backend, &self.output)?; for (node, path) in RafsIterator::new(&rafs) { builder.append(&*node, &path)?; @@ -114,13 +115,13 @@ impl OCITarBuilderFactory { fn create( &self, meta: &RafsSuper, - blob_path: Option<&Path>, + blob_backend: &Option>, output_path: &Path, ) -> Result> { let writer = self.create_writer(output_path)?; let blob = meta.superblock.get_blob_infos().pop(); - let builders = self.create_builders(blob, blob_path)?; + let builders = self.create_builders(blob, blob_backend)?; let builder = OCITarBuilder::new(builders, writer); @@ -144,7 +145,7 @@ impl OCITarBuilderFactory { fn create_builders( &self, blob: Option>, - blob_path: Option<&Path>, + blob_backend: &Option>, ) -> Result>> { // PAX basic builders let ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); @@ -159,7 +160,7 @@ impl OCITarBuilderFactory { let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); let char_builder = OCICharBuilder::new(special_builder.clone()); let block_builder = OCIBlockBuilder::new(special_builder); - let reg_builder = self.create_reg_builder(blob, blob_path)?; + let reg_builder = self.create_reg_builder(blob, blob_backend)?; // The order counts. let builders = vec![ @@ -179,16 +180,18 @@ impl OCITarBuilderFactory { fn create_reg_builder( &self, blob: Option>, - blob_path: Option<&Path>, + blob_backend: &Option>, ) -> Result { let (reader, compressor) = match blob { None => (None, None), Some(ref blob) => { - if blob_path.is_none() { - bail!("miss blob path") - } + let blob_backend = blob_backend + .as_deref() + .with_context(|| "both blob path or blob backend config are not specified")?; + let reader = blob_backend + .get_reader("unpacker") + .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; - let reader = self.create_blob_reader(blob_path.unwrap())?; let compressor = blob.compressor(); (Some(reader), Some(compressor)) @@ -201,23 +204,6 @@ impl OCITarBuilderFactory { compressor, )) } - - fn create_blob_reader(&self, blob_path: &Path) -> Result> { - let config = LocalFsConfig { - blob_file: blob_path.to_str().unwrap().to_owned(), - dir: Default::default(), - alt_dirs: Default::default(), - }; - - let backend = LocalFs::new(&config, Some("unpacker")) - .with_context(|| format!("fail to create local backend for {:?}", blob_path))?; - - let reader = backend - .get_reader("") - .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; - - Ok(reader) - } } struct OCITarBuilder {