Skip to content

Commit

Permalink
Add backend-config support to nydus-image unpack
Browse files Browse the repository at this point in the history 
The `nydus-image unpack` only support reading nydus blob from the blob files on local machine (by the argument `blob`) by now. This patch adds a new argument `backend-config` to this command to allow `nydus-image unpack` to read blob data from all kinds of backends.

Signed-off-by: Nan Li <loheagn@icloud.com>
  • Loading branch information
@loheagn
loheagn committed Feb 16, 2023
1 parent e057895 commit 10fedc2
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 44 deletions.
10 changes: 10 additions & 0 deletions api/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1878,6 +1878,16 @@ mod tests {
assert_eq!(&config.id, "");
}

#[test]
fn test_backend_http_proxy_config() {
let config =
r#"{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"/tmp"}}}"#;
let config = ConfigV2::from_str(config).unwrap();
let backend = config.backend.unwrap();
assert_eq!(&backend.backend_type, "http-proxy");
assert_eq!(&backend.http_proxy.unwrap().addr, "/tmp");
}

#[test]
fn test_new_localfs() {
let config = ConfigV2::new_localfs("id1", "./").unwrap();
Expand Down
47 changes: 41 additions & 6 deletions src/bin/nydus-image/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ use clap::parser::ValueSource;
use clap::{Arg, ArgAction, ArgMatches, Command as App};
use nix::unistd::{getegid, geteuid};
use nydus::get_build_time_info;
use nydus_api::{BuildTimeInfo, ConfigV2};
use nydus_api::{BuildTimeInfo, ConfigV2, LocalFsConfig};
use nydus_app::setup_logging;
use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion};
use nydus_storage::backend::localfs::LocalFs;
use nydus_storage::backend::BlobBackend;
use nydus_storage::device::BlobFeatures;
use nydus_storage::factory::BlobFactory;
use nydus_storage::meta::format_blob_features;
Expand Down Expand Up @@ -525,6 +527,12 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
.help("File path of RAFS metadata")
.required_unless_present("bootstrap"),
)
.arg(
Arg::new("backend-config")
.long("backend-config")
.help("config file of backend")
.required(false),
)
.arg(
Arg::new("bootstrap")
.short('B')
Expand Down Expand Up @@ -955,10 +963,7 @@ impl Command {
Some(args) => Some(import_chunk_dict(args, config, &rs.meta.get_config())?),
};

let cfg_file = matches.get_one::<String>("backend-config").unwrap();
let cfg = ConfigV2::from_file(cfg_file)?;
let backend_cfg = cfg.get_backend_config()?;
let backend = BlobFactory::new_backend(backend_cfg, "compactor")?;
let backend = Self::get_backend(matches, "compactor")?;

let config_file_path = matches.get_one::<String>("config").unwrap();
let file = File::open(config_file_path)
Expand All @@ -984,9 +989,25 @@ impl Command {
if output.is_empty() {
return Err(anyhow!("invalid empty --output option"));
}

let blob = matches.get_one::<String>("blob").map(|s| s.as_str());
let backend: Option<Arc<dyn BlobBackend + Send + Sync>> = match blob {
Some(blob_path) => {
let blob_path = PathBuf::from(blob_path);
let local_fs_conf = LocalFsConfig {
blob_file: blob_path.to_str().unwrap().to_owned(),
dir: Default::default(),
alt_dirs: Default::default(),
};
let local_fs = LocalFs::new(&local_fs_conf, Some("unpacker"))
.with_context(|| format!("fail to create local backend for {:?}", blob_path))?;

Some(Arc::new(local_fs))
}
None => Self::get_backend(matches, "unpacker").ok(),
};

OCIUnpacker::new(bootstrap, blob, output)
OCIUnpacker::new(bootstrap, backend, output)
.with_context(|| "fail to create unpacker")?
.unpack(config)
.with_context(|| "fail to unpack")
Expand Down Expand Up @@ -1210,6 +1231,20 @@ impl Command {
Ok(Arc::new(config))
}

fn get_backend(
matches: &ArgMatches,
blob_id: &str,
) -> Result<Arc<dyn BlobBackend + Send + Sync>> {
let cfg_file = matches
.get_one::<String>("backend-config")
.context("missing backend-config argument")?;
let cfg = ConfigV2::from_file(cfg_file)?;
let backend_cfg = cfg.get_backend_config()?;
let backend = BlobFactory::new_backend(backend_cfg, blob_id)?;

Ok(backend)
}

fn get_blob_id(matches: &ArgMatches) -> Result<String> {
let mut blob_id = String::new();

Expand Down
62 changes: 24 additions & 38 deletions src/bin/nydus-image/unpack/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,13 @@ use std::str;
use std::sync::Arc;

use anyhow::{Context, Result};
use nydus_api::{ConfigV2, LocalFsConfig};
use nydus_api::ConfigV2;
use nydus_rafs::{
metadata::{RafsInodeExt, RafsSuper},
RafsIterator,
};
use nydus_storage::{
backend::{localfs::LocalFs, BlobBackend, BlobReader},
device::BlobInfo,
};
use nydus_storage::backend::BlobBackend;
use nydus_storage::device::BlobInfo;
use tar::{Builder, Header};

use self::pax::{
Expand All @@ -36,24 +34,27 @@ pub trait Unpacker {
/// A unpacker with the ability to convert bootstrap file and blob file to tar
pub struct OCIUnpacker {
bootstrap: PathBuf,
blob: Option<PathBuf>,
blob_backend: Option<Arc<dyn BlobBackend + Send + Sync>>,
output: PathBuf,

builder_factory: OCITarBuilderFactory,
}

impl OCIUnpacker {
pub fn new(bootstrap: &Path, blob: Option<&str>, output: &str) -> Result<Self> {
pub fn new(
bootstrap: &Path,
blob_backend: Option<Arc<dyn BlobBackend + Send + Sync>>,
output: &str,
) -> Result<Self> {
let bootstrap = bootstrap.to_path_buf();
let output = PathBuf::from(output);
let blob = blob.map(PathBuf::from);

let builder_factory = OCITarBuilderFactory::new();

Ok(OCIUnpacker {
builder_factory,
bootstrap,
blob,
blob_backend,
output,
})
}
Expand All @@ -72,15 +73,15 @@ impl OCIUnpacker {
impl Unpacker for OCIUnpacker {
fn unpack(&self, config: Arc<ConfigV2>) -> Result<()> {
debug!(
"oci unpacker, bootstrap file: {:?}, blob file: {:?}, output file: {:?}",
self.bootstrap, self.blob, self.output
"oci unpacker, bootstrap file: {:?}, output file: {:?}",
self.bootstrap, self.output
);

let rafs = self.load_rafs(config)?;

let mut builder = self
.builder_factory
.create(&rafs, self.blob.as_deref(), &self.output)?;
.create(&rafs, &self.blob_backend, &self.output)?;

for (node, path) in RafsIterator::new(&rafs) {
builder.append(&*node, &path)?;
Expand Down Expand Up @@ -114,13 +115,13 @@ impl OCITarBuilderFactory {
fn create(
&self,
meta: &RafsSuper,
blob_path: Option<&Path>,
blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
output_path: &Path,
) -> Result<Box<dyn TarBuilder>> {
let writer = self.create_writer(output_path)?;

let blob = meta.superblock.get_blob_infos().pop();
let builders = self.create_builders(blob, blob_path)?;
let builders = self.create_builders(blob, blob_backend)?;

let builder = OCITarBuilder::new(builders, writer);

Expand All @@ -144,7 +145,7 @@ impl OCITarBuilderFactory {
fn create_builders(
&self,
blob: Option<Arc<BlobInfo>>,
blob_path: Option<&Path>,
blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
) -> Result<Vec<Box<dyn SectionBuilder>>> {
// PAX basic builders
let ext_builder = Rc::new(PAXExtensionSectionBuilder::new());
Expand All @@ -159,7 +160,7 @@ impl OCITarBuilderFactory {
let fifo_builder = OCIFifoBuilder::new(special_builder.clone());
let char_builder = OCICharBuilder::new(special_builder.clone());
let block_builder = OCIBlockBuilder::new(special_builder);
let reg_builder = self.create_reg_builder(blob, blob_path)?;
let reg_builder = self.create_reg_builder(blob, blob_backend)?;

// The order counts.
let builders = vec![
Expand All @@ -179,16 +180,18 @@ impl OCITarBuilderFactory {
fn create_reg_builder(
&self,
blob: Option<Arc<BlobInfo>>,
blob_path: Option<&Path>,
blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
) -> Result<OCIRegBuilder> {
let (reader, compressor) = match blob {
None => (None, None),
Some(ref blob) => {
if blob_path.is_none() {
bail!("miss blob path")
}
let blob_backend = blob_backend
.as_deref()
.with_context(|| "both blob path or blob backend config are not specified")?;
let reader = blob_backend
.get_reader("unpacker")
.map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;

let reader = self.create_blob_reader(blob_path.unwrap())?;
let compressor = blob.compressor();

(Some(reader), Some(compressor))
Expand All @@ -201,23 +204,6 @@ impl OCITarBuilderFactory {
compressor,
))
}

fn create_blob_reader(&self, blob_path: &Path) -> Result<Arc<dyn BlobReader>> {
let config = LocalFsConfig {
blob_file: blob_path.to_str().unwrap().to_owned(),
dir: Default::default(),
alt_dirs: Default::default(),
};

let backend = LocalFs::new(&config, Some("unpacker"))
.with_context(|| format!("fail to create local backend for {:?}", blob_path))?;

let reader = backend
.get_reader("")
.map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;

Ok(reader)
}
}

struct OCITarBuilder {
Expand Down

0 comments on commit 10fedc2

Please sign in to comment.