[feat] add nvme support

Makefile

@@ -20,6 +20,7 @@
 #     - `QEMU_LOG`: Enable QEMU logging (log file is "qemu.log")
 #     - `NET_DUMP`: Enable network packet dump (log file is "netdump.pcap")
 #     - `NET_DEV`: QEMU netdev backend types: user, tap, bridge
+#     - `BLK_DEV`: QEMU blkdev backend types: virtio, nvme
 #     - `VFIO_PCI`: PCI device address in the format "bus:dev.func" to passthrough
 #     - `VHOST`: Enable vhost-net for tap backend (only for `NET_DEV=tap`)
 # * Network options:
@@ -51,6 +52,7 @@ DISK_IMG ?= disk.img
 QEMU_LOG ?= n
 NET_DUMP ?= n
 NET_DEV ?= user
+BLK_DEV ?= virtio
 VFIO_PCI ?=
 VHOST ?= n
 

api/axfeat/Cargo.toml

@@ -55,6 +55,7 @@ bus-pci = ["axdriver?/bus-pci"]
 driver-ramdisk = ["axdriver?/ramdisk", "axfs?/use-ramdisk"]
 driver-ixgbe = ["axdriver?/ixgbe"]
 driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"]
+driver-nvme = ["axdriver?/nvme"]
 
 # Logging
 log-level-off = ["axlog/log-level-off"]

examples/shell/README.md

@@ -0,0 +1,10 @@
+# Useage
+
+## Example
+
+use nvme as block device.
+
+```bash
+make disk_img
+make A=examples/shell ARCH=x86_64 LOG=info SMP=4 ACCEL=N FEATURES=driver-nvme APP_FEATURES=use-ramfs BLK=y run
+```

modules/axdriver/Cargo.toml

@@ -2,7 +2,10 @@
 name = "axdriver"
 version.workspace = true
 edition = "2021"
-authors = ["Yuekai Jia <equation618@gmail.com>", "ChengXiang Qi <kuangjux@outlook.com>"]
+authors = [
+    "Yuekai Jia <equation618@gmail.com>",
+    "ChengXiang Qi <kuangjux@outlook.com>",
+]
 description = "ArceOS device drivers"
 license.workspace = true
 homepage.workspace = true
@@ -12,7 +15,14 @@ documentation = "https://arceos-org.github.io/arceos/axdriver/index.html"
 [features]
 dyn = []
 bus-mmio = []
-bus-pci = ["dep:axdriver_pci", "dep:axhal", "dep:axconfig"]
+bus-pci = [
+    "dep:axdriver_pci",
+    "dep:axhal",
+    "dep:axalloc",
+    "dep:axconfig",
+    "dep:pcie",
+    "axhal/alloc",
+]
 net = ["axdriver_net"]
 block = ["axdriver_block"]
 display = ["axdriver_display"]
@@ -27,6 +37,7 @@ virtio-gpu = ["display", "virtio", "axdriver_virtio/gpu"]
 ramdisk = ["block", "axdriver_block/ramdisk"]
 bcm2835-sdhci = ["block", "axdriver_block/bcm2835-sdhci"]
 ixgbe = ["net", "axdriver_net/ixgbe", "dep:axalloc", "dep:axhal", "dep:axdma"]
+nvme = ["block", "bus-pci", "dep:nvme-driver", "dep:axalloc"]
 # more devices example: e1000 = ["net", "axdriver_net/e1000"]
 
 default = ["bus-pci"]
@@ -43,4 +54,6 @@ axdriver_virtio = { git = "https://github.com/arceos-org/axdriver_crates.git", t
 axalloc = { workspace = true, optional = true }
 axhal = { workspace = true, optional = true }
 axconfig = { workspace = true, optional = true }
-axdma = { workspace = true, optional = true }
+axdma = { workspace = true, optional = true }
+pcie = { version = "0.2", optional = true }
+nvme-driver = { version = "0.2", optional = true }

modules/axdriver/build.rs

@@ -1,5 +1,5 @@
 const NET_DEV_FEATURES: &[&str] = &["ixgbe", "virtio-net"];
-const BLOCK_DEV_FEATURES: &[&str] = &["ramdisk", "bcm2835-sdhci", "virtio-blk"];
+const BLOCK_DEV_FEATURES: &[&str] = &["nvme", "ramdisk", "bcm2835-sdhci", "virtio-blk"];
 const DISPLAY_DEV_FEATURES: &[&str] = &["virtio-gpu"];
 
 fn make_cfg_values(str_list: &[&str]) -> String {

modules/axdriver/src/bus/pci.rs

@@ -1,8 +1,11 @@
+use core::ptr::NonNull;
+
 use crate::{prelude::*, AllDevices};
 use axdriver_pci::{
     BarInfo, Cam, Command, DeviceFunction, HeaderType, MemoryBarType, PciRangeAllocator, PciRoot,
 };
 use axhal::mem::phys_to_virt;
+use pcie::{Header, RootComplexGeneric};
 
 const PCI_BAR_NUM: u8 = 6;
 
@@ -84,6 +87,33 @@ fn config_pci_device(
 
 impl AllDevices {
     pub(crate) fn probe_bus_devices(&mut self) {
+        self.probe_pci_devices();
+        self.probe_pcie_devices();
+    }
+
+    fn probe_pcie_devices(&mut self) {
+        let base_vaddr = phys_to_virt(axconfig::PCI_ECAM_BASE.into());
+        let mut root = RootComplexGeneric::new(NonNull::new(base_vaddr.as_mut_ptr()).unwrap());
+
+        for elem in root.enumerate_keep_bar(None) {
+            if let Header::Endpoint(ep) = elem.header {
+                for_each_drivers!(type Driver, {
+                    if let Some(dev) = Driver::probe_pcie(elem.root, &ep) {
+                        info!(
+                            "registered a new {:?} device at {:?}: {:?}",
+                            dev.device_type(),
+                            ep.address,
+                            dev.device_name(),
+                        );
+                        self.add_device(dev);
+                        continue; // skip to the next device
+                    }
+                })
+            }
+        }
+    }
+
+    fn probe_pci_devices(&mut self) {
         let base_vaddr = phys_to_virt(axconfig::PCI_ECAM_BASE.into());
         let mut root = unsafe { PciRoot::new(base_vaddr.as_mut_ptr(), Cam::Ecam) };
 

modules/axdriver/src/drivers.rs

@@ -31,6 +31,14 @@ pub trait DriverProbe {
     ) -> Option<AxDeviceEnum> {
         None
     }
+
+    #[cfg(bus = "pci")]
+    fn probe_pcie(
+        _root: &mut pcie::RootComplexGeneric,
+        _ep: &pcie::Endpoint,
+    ) -> Option<AxDeviceEnum> {
+        None
+    }
 }
 
 #[cfg(net_dev = "virtio-net")]
@@ -128,3 +136,23 @@ cfg_if::cfg_if! {
         }
     }
 }
+
+cfg_if::cfg_if! {
+    if #[cfg(block_dev="nvme")] {
+        use crate::nvme::Nvme;
+        use pcie::{Endpoint, RootComplexGeneric};
+        pub struct NvmeDriver;
+        register_block_driver!(NvmeDriver, Nvme);
+
+        impl DriverProbe for NvmeDriver {
+            #[cfg(bus = "pci")]
+            fn probe_pcie(root: &mut RootComplexGeneric, ep: &Endpoint) -> Option<crate::AxDeviceEnum> {
+                use crate::AxDeviceEnum;
+
+                let dev = Nvme::new(root, ep)?;
+
+                Some(AxDeviceEnum::from_block(dev))
+            }
+        }
+    }
+}

modules/axdriver/src/lib.rs

@@ -78,6 +78,9 @@ mod virtio;
 #[cfg(feature = "ixgbe")]
 mod ixgbe;
 
+#[cfg(feature = "nvme")]
+mod nvme;
+
 pub mod prelude;
 
 #[allow(unused_imports)]

modules/axdriver/src/macros.rs

@@ -64,5 +64,10 @@ macro_rules! for_each_drivers {
             type $drv_type = crate::drivers::IxgbeDriver;
             $code
         }
+        #[cfg(block_dev = "nvme")]
+        {
+            type $drv_type = crate::drivers::NvmeDriver;
+            $code
+        }
     }};
 }

modules/axdriver/src/nvme.rs

@@ -0,0 +1,96 @@
+use core::ptr::NonNull;
+
+use axhal::mem::phys_to_virt;
+use nvme_driver::{Config, Namespace};
+use pcie::{Chip, CommandRegister, DeviceType, Endpoint, RootComplex};
+
+use crate::{BaseDriverOps, BlockDriverOps};
+
+pub struct Nvme {
+    inner: nvme_driver::Nvme,
+    ns: Namespace,
+}
+
+unsafe impl Send for Nvme {}
+unsafe impl Sync for Nvme {}
+
+impl Nvme {
+    pub fn new<C: Chip>(root: &mut RootComplex<C>, ep: &Endpoint) -> Option<Self> {
+        ep.update_command(root, |cmd| {
+            cmd | CommandRegister::IO_ENABLE
+                | CommandRegister::MEMORY_ENABLE
+                | CommandRegister::BUS_MASTER_ENABLE
+        });
+
+        if ep.device_type() == DeviceType::NvmeController {
+            let bar_addr = match &ep.bar {
+                pcie::BarVec::Memory32(bar_vec_t) => {
+                    let bar0 = bar_vec_t[0].as_ref().unwrap();
+                    bar0.address as usize
+                }
+                pcie::BarVec::Memory64(bar_vec_t) => {
+                    let bar0 = bar_vec_t[0].as_ref().unwrap();
+                    bar0.address as usize
+                }
+                pcie::BarVec::Io(_bar_vec_t) => return None,
+            };
+
+            let addr = phys_to_virt(bar_addr.into());
+
+            let mut nvme = nvme_driver::Nvme::new(
+                unsafe { NonNull::new_unchecked(addr.as_mut_ptr()) },
+                Config {
+                    page_size: 0x1000,
+                    io_queue_pair_count: 1,
+                },
+            )
+            .inspect_err(|e| error!("{:?}", e))
+            .unwrap();
+            let ns_list = nvme.namespace_list().ok()?;
+            let ns = ns_list.first()?;
+
+            return Some(Self {
+                inner: nvme,
+                ns: *ns,
+            });
+        }
+
+        None
+    }
+}
+
+impl BaseDriverOps for Nvme {
+    fn device_name(&self) -> &str {
+        "NVME"
+    }
+
+    fn device_type(&self) -> crate::DeviceType {
+        crate::DeviceType::Block
+    }
+}
+
+impl BlockDriverOps for Nvme {
+    fn num_blocks(&self) -> u64 {
+        self.ns.lba_count as _
+    }
+
+    fn block_size(&self) -> usize {
+        self.ns.lba_size
+    }
+
+    fn read_block(&mut self, block_id: u64, buf: &mut [u8]) -> crate::DevResult {
+        self.inner
+            .block_read_sync(&self.ns, block_id as _, buf)
+            .map_err(|_e| crate::DevError::Io)
+    }
+
+    fn write_block(&mut self, block_id: u64, buf: &[u8]) -> crate::DevResult {
+        self.inner
+            .block_write_sync(&self.ns, block_id, buf)
+            .map_err(|_e| crate::DevError::Io)
+    }
+
+    fn flush(&mut self) -> crate::DevResult {
+        Ok(())
+    }
+}

modules/axhal/Cargo.toml

@@ -37,6 +37,7 @@ page_table_multiarch = { version = "0.4", optional = true }
 axlog = { workspace = true }
 axconfig = { workspace = true }
 axalloc = { workspace = true, optional = true }
+dma-api = { version = "0.1" }
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
 x86 = "0.52"

modules/axhal/src/arch/aarch64/cache.rs

@@ -0,0 +1,53 @@
+#![allow(unused)]
+
+use core::{arch::asm, ptr::NonNull};
+
+#[naked]
+unsafe extern "C" fn _dcache_invalidate_range(_addr: usize, _end: usize) {
+    asm!(
+        "mrs	x3, ctr_el0",
+        "ubfx	x3, x3, #16, #4",
+        "mov	x2, #4",
+        "lsl	x2, x2, x3", /* cache line size */
+        /* x2 <- minimal cache line size in cache system */
+        "sub	x3, x2, #1",
+        "bic	x0, x0, x3",
+        "1:	dc	ivac, x0", /* invalidate data or unified cache */
+        "add	x0, x0, x2",
+        "cmp	x0, x1",
+        "b.lo	1b",
+        "dsb	sy",
+        "ret",
+        options(noreturn)
+    );
+}
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(addr: NonNull<u8>, size: usize) {
+    unsafe { _dcache_invalidate_range(addr.as_ptr() as usize, addr.as_ptr() as usize + size) }
+}
+
+#[naked]
+unsafe extern "C" fn _dcache_flush_range(_addr: usize, _end: usize) {
+    asm!(
+        "mrs	x3, ctr_el0",
+        "ubfx	x3, x3, #16, #4",
+        "mov	x2, #4",
+        "lsl	x2, x2, x3", /* cache line size */
+        /* x2 <- minimal cache line size in cache system */
+        "sub	x3, x2, #1",
+        "bic	x0, x0, x3",
+        "1:	dc	civac, x0", /* clean & invalidate data or unified cache */
+        "add	x0, x0, x2",
+        "cmp	x0, x1",
+        "b.lo	1b",
+        "dsb	sy",
+        "ret",
+        options(noreturn)
+    );
+}
+
+/// Flush data cache
+pub fn dcache_flush_range(addr: NonNull<u8>, size: usize) {
+    unsafe { _dcache_flush_range(addr.as_ptr() as usize, addr.as_ptr() as usize + size) }
+}

modules/axhal/src/arch/aarch64/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod cache;
 mod context;
 pub(crate) mod trap;
 

modules/axhal/src/arch/riscv/cache.rs

@@ -0,0 +1,13 @@
+#![allow(unused)]
+
+use core::ptr::NonNull;
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(_addr: NonNull<u8>, _size: usize) {
+    unimplemented!();
+}
+
+/// Flush data cache
+pub fn dcache_flush_range(_addr: NonNull<u8>, _size: usize) {
+    unimplemented!();
+}

modules/axhal/src/arch/riscv/mod.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 mod macros;
 
+pub mod cache;
 mod context;
 mod trap;
 

modules/axhal/src/arch/x86_64/cache.rs

@@ -0,0 +1,13 @@
+#![allow(unused)]
+
+use core::ptr::NonNull;
+
+/// Invalidate data cache
+pub fn dcache_invalidate_range(_addr: NonNull<u8>, _size: usize) {
+    unimplemented!();
+}
+
+/// Flush data cache
+pub fn dcache_flush_range(_addr: NonNull<u8>, _size: usize) {
+    unimplemented!();
+}

modules/axhal/src/arch/x86_64/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod cache;
 mod context;
 mod gdt;
 mod idt;