Skip to content

Commit

Permalink
service: enhance block device to support RAFS filesystem in TARFS mode
Browse files Browse the repository at this point in the history 
Enhance block device to support block size of 512, in addition to 4096,
so we can expose RAFS filesystems in TARFS mode as block device.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>
  • Loading branch information
@jiangliu
jiangliu committed Mar 16, 2023
1 parent 6522750 commit 3891a51
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 27 deletions.
12 changes: 11 additions & 1 deletion service/src/blob_cache.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use nydus_api::{
BLOB_CACHE_TYPE_META_BLOB,
};
use nydus_rafs::metadata::layout::v6::{EROFS_BLOCK_BITS_12, EROFS_BLOCK_SIZE_4096};
use nydus_rafs::metadata::{RafsBlobExtraInfo, RafsSuper};
use nydus_rafs::metadata::{RafsBlobExtraInfo, RafsSuper, RafsSuperFlags};
use nydus_storage::cache::BlobCache;
use nydus_storage::device::BlobInfo;
use nydus_storage::factory::BLOB_FACTORY;
Expand All @@ -43,6 +43,7 @@ pub struct MetaBlobConfig {
config: Arc<ConfigV2>,
blobs: Mutex<Vec<Arc<DataBlobConfig>>>,
blob_extra_infos: HashMap<String, RafsBlobExtraInfo>,
is_tarfs_mode: bool,
}

impl MetaBlobConfig {
Expand Down Expand Up @@ -70,6 +71,11 @@ impl MetaBlobConfig {
self.blob_extra_infos.get(blob_id)
}

/// Check whether the filesystem is in `TARFS` mode.
pub fn is_tarfs_mode(&self) -> bool {
self.is_tarfs_mode
}

fn add_data_blob(&self, blob: Arc<DataBlobConfig>) {
self.blobs.lock().unwrap().push(blob);
}
Expand Down Expand Up @@ -130,6 +136,7 @@ impl BlobConfig {
path: PathBuf,
config: Arc<ConfigV2>,
blob_extra_infos: HashMap<String, RafsBlobExtraInfo>,
is_tarfs_mode: bool,
) -> Self {
let scoped_blob_id = generate_blob_key(&domain_id, &blob_id);

Expand All @@ -140,6 +147,7 @@ impl BlobConfig {
config,
blobs: Mutex::new(Vec::new()),
blob_extra_infos,
is_tarfs_mode,
}))
}

Expand Down Expand Up @@ -394,6 +402,7 @@ impl BlobCacheMgr {
path,
config,
blob_extra_infos,
rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE),
);
// Safe to unwrap because it's a meta blob object.
let meta_obj = meta.meta_config().unwrap();
Expand Down Expand Up @@ -612,6 +621,7 @@ mod tests {
config,
blobs: Mutex::new(Vec::new()),
blob_extra_infos: HashMap::new(),
is_tarfs_mode: false,
};
assert_eq!(blob.path(), &path);
}
Expand Down
75 changes: 61 additions & 14 deletions service/src/block_device.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ use std::io::Result;
use std::sync::Arc;

use dbs_allocator::{Constraint, IntervalTree, NodeState, Range};
use nydus_rafs::metadata::layout::v6::EROFS_BLOCK_BITS_12;
use nydus_rafs::metadata::layout::v6::{
EROFS_BLOCK_BITS_12, EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512,
};
use tokio_uring::buf::IoBufMut;

use crate::blob_cache::{BlobCacheMgr, BlobConfig, DataBlob, MetaBlob};
Expand All @@ -40,6 +42,7 @@ pub struct BlockDevice {
blob_id: String,
cache_mgr: Arc<BlobCacheMgr>,
ranges: IntervalTree<BlockRange>,
is_tarfs_mode: bool,
}

impl BlockDevice {
Expand All @@ -63,11 +66,15 @@ impl BlockDevice {
}
Some(BlobConfig::MetaBlob(v)) => v,
};
let is_tarfs_mode = meta_blob_config.is_tarfs_mode();
let meta_blob = MetaBlob::new(meta_blob_config.path())?;
let meta_blob = Arc::new(meta_blob);
let constraint = Constraint::new(meta_blob.blocks())
.min(0u32)
.max(meta_blob.blocks() - 1);
let blocks = if is_tarfs_mode {
meta_blob.blocks() * 8
} else {
meta_blob.blocks()
};
let constraint = Constraint::new(blocks).min(0u32).max(blocks - 1);
let range = ranges.allocate(&constraint).ok_or_else(|| {
enoent!(format!(
"block_device: failed to allocate address range for meta blob {}",
Expand All @@ -76,7 +83,7 @@ impl BlockDevice {
})?;
ranges.update(&range, BlockRange::MetaBlob(meta_blob.clone()));

let mut pos = meta_blob.blocks();
let mut pos = blocks;
let data_blobs = meta_blob_config.get_blobs();
for blob in data_blobs.iter() {
let blob_info = blob.blob_info();
Expand All @@ -97,6 +104,13 @@ impl BlockDevice {
);
return Err(einval!(msg));
}
if is_tarfs_mode != blob_info.features().is_tarfs() {
let msg = format!(
"block_device: inconsistent `TARFS` mode from meta and data blob {}",
blob_id
);
return Err(einval!(msg));
}

if pos < extra_info.mapped_blkaddr {
let constraint = Constraint::new(extra_info.mapped_blkaddr - pos)
Expand All @@ -108,7 +122,11 @@ impl BlockDevice {
ranges.update(&range, BlockRange::Hole);
}

let blocks = blob_info.uncompressed_size() >> EROFS_BLOCK_BITS_12;
let blocks = if is_tarfs_mode {
blob_info.uncompressed_size() >> EROFS_BLOCK_BITS_9
} else {
blob_info.uncompressed_size() >> EROFS_BLOCK_BITS_12
};
if blocks > u32::MAX as u64
|| blocks + extra_info.mapped_blkaddr as u64 > u32::MAX as u64
{
Expand Down Expand Up @@ -137,6 +155,7 @@ impl BlockDevice {
blob_id,
cache_mgr,
ranges,
is_tarfs_mode,
})
}

Expand All @@ -155,23 +174,49 @@ impl BlockDevice {
self.blocks
}

/// Get block size of block device.
pub fn block_size(&self) -> u64 {
if self.is_tarfs_mode {
EROFS_BLOCK_SIZE_512
} else {
EROFS_BLOCK_SIZE_4096
}
}

/// Convert data size to number of blocks.
pub fn size_to_blocks(&self, sz: u64) -> u64 {
if self.is_tarfs_mode {
sz >> EROFS_BLOCK_BITS_9
} else {
sz >> EROFS_BLOCK_BITS_12
}
}

/// Convert number of blocks to data size.
pub fn blocks_to_size(&self, blocks: u32) -> u64 {
if self.is_tarfs_mode {
(blocks as u64) << EROFS_BLOCK_BITS_9
} else {
(blocks as u64) << EROFS_BLOCK_BITS_12
}
}

/// Read block range [start, start + blocks) from the block device.
pub async fn async_read<T: IoBufMut>(
&self,
mut start: u32,
mut blocks: u32,
mut buf: T,
) -> (Result<usize>, T) {
if start.checked_add(blocks).is_none()
|| (blocks as u64) << EROFS_BLOCK_BITS_12 > buf.bytes_total() as u64
{
let sz = self.blocks_to_size(blocks);
if start.checked_add(blocks).is_none() || sz > buf.bytes_total() as u64 {
return (
Err(einval!("block_device: invalid parameters to read()")),
buf,
);
}

let total_size = (blocks as usize) << EROFS_BLOCK_BITS_12;
let total_size = sz as usize;
let mut pos = 0;
while blocks > 0 {
let (range, node) = match self.ranges.get_superset(&Range::new_point(start)) {
Expand All @@ -189,19 +234,21 @@ impl BlockDevice {

if let NodeState::Valued(r) = node {
let count = min(range.max as u32 - start + 1, blocks);
let sz = (count as usize) << EROFS_BLOCK_BITS_12 as usize;
let sz = self.blocks_to_size(count) as usize;
let mut s = buf.slice(pos..pos + sz);
let (res, s) = match r {
BlockRange::Hole => {
s.fill(0);
(Ok(sz), s)
}
BlockRange::MetaBlob(m) => {
m.async_read((start as u64) << EROFS_BLOCK_BITS_12, s).await
let offset = self.blocks_to_size(start);
m.async_read(offset, s).await
}
BlockRange::DataBlob(d) => {
let offset = start as u64 - range.min;
d.async_read(offset << EROFS_BLOCK_BITS_12, s).await
let offset = start - range.min as u32;
let offset = self.blocks_to_size(offset);
d.async_read(offset, s).await
}
};

Expand Down
17 changes: 5 additions & 12 deletions service/src/block_nbd.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ use std::thread::JoinHandle;
use bytes::{Buf, BufMut};
use mio::Waker;
use nydus_api::{BlobCacheEntry, BuildTimeInfo};
use nydus_rafs::metadata::layout::v6::{EROFS_BLOCK_BITS_12, EROFS_BLOCK_SIZE_4096};
use nydus_storage::utils::alloc_buf;
use tokio::sync::broadcast::{channel, Sender};
use tokio_uring::buf::IoBuf;
Expand Down Expand Up @@ -85,11 +84,7 @@ impl NbdService {
error!("block_nbd: failed to open NBD device {}", nbd_path);
e
})?;
nbd_ioctl(
nbd_dev.as_raw_fd(),
NBD_SET_BLOCK_SIZE,
EROFS_BLOCK_SIZE_4096,
)?;
nbd_ioctl(nbd_dev.as_raw_fd(), NBD_SET_BLOCK_SIZE, device.block_size())?;
nbd_ioctl(nbd_dev.as_raw_fd(), NBD_SET_BLOCKS, device.blocks() as u64)?;
nbd_ioctl(nbd_dev.as_raw_fd(), NBD_SET_TIMEOUT, 60)?;
nbd_ioctl(nbd_dev.as_raw_fd(), NBD_CLEAR_SOCK, 0)?;
Expand Down Expand Up @@ -226,20 +221,18 @@ impl NbdWorker {
let pos = request.get_u64();
let len = request.get_u32();

let block_size = device.block_size();
let mut code = NBD_OK;
let mut data_buf = alloc_buf(len as usize);
if magic != NBD_REQUEST_MAGIC
|| pos % EROFS_BLOCK_SIZE_4096 != 0
|| len as u64 % EROFS_BLOCK_SIZE_4096 != 0
{
if magic != NBD_REQUEST_MAGIC || pos % block_size != 0 || len as u64 % block_size != 0 {
warn!(
"block_nbd: invalid request magic 0x{:x}, type {}, pos 0x{:x}, len 0x{:x}",
magic, ty, pos, len
);
code = NBD_EINVAL;
} else if ty == NBD_CMD_READ {
let start = (pos >> EROFS_BLOCK_BITS_12) as u32;
let count = len >> EROFS_BLOCK_BITS_12;
let start = (pos / block_size) as u32;
let count = len / block_size as u32;
let (res, buf) = device.async_read(start, count, data_buf).await;
data_buf = buf;
match res {
Expand Down

0 comments on commit 3891a51

Please sign in to comment.