From 733cc49c494272d7714d421df05d9e73d9b81080 Mon Sep 17 00:00:00 2001 From: Dion Dokter Date: Fri, 12 Jan 2024 16:20:49 +0100 Subject: [PATCH] Added the first real cache --- fuzz/fuzz_targets/queue.rs | 8 ++ src/cache.rs | 199 ++++++++++++++++++++++++++++++++----- src/lib.rs | 53 +++++++--- src/map.rs | 26 +---- src/queue.rs | 16 +-- 5 files changed, 223 insertions(+), 79 deletions(-) diff --git a/fuzz/fuzz_targets/queue.rs b/fuzz/fuzz_targets/queue.rs index 91a46e2..eb59538 100644 --- a/fuzz/fuzz_targets/queue.rs +++ b/fuzz/fuzz_targets/queue.rs @@ -89,6 +89,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -149,6 +150,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -198,6 +200,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -224,6 +227,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -273,6 +277,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -308,6 +313,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -334,6 +340,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; @@ -365,6 +372,7 @@ fn fuzz(ops: Input) { block_on(sequential_storage::queue::try_repair( &mut flash, FLASH_RANGE, + &mut cache, )) .unwrap(); corruption_repaired = true; diff --git a/src/cache.rs b/src/cache.rs index a59f159..a6a2042 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -15,36 +15,11 @@ pub(crate) trait StateQuery { fn unmark_dirty(&mut self); fn is_dirty(&self) -> bool; - fn notice_page_state(&mut self, page_index: usize, new_state: PageState); + fn notice_page_state(&mut self, _page_index: usize, _new_state: PageState) {} /// Get the state of the page located at the given index async fn get_page_state( - &self, - flash: &mut S, - flash_range: Range, - page_index: usize, - ) -> Result>; -} - -pub struct NoCache; - -impl StateQuery for NoCache { - fn invalidate_cache_state(&mut self) {} - - fn mark_dirty(&mut self) {} - - fn unmark_dirty(&mut self) {} - - fn is_dirty(&self) -> bool { - false - } - - fn notice_page_state(&mut self, _page_index: usize, _new_state: PageState) { - todo!() - } - - async fn get_page_state( - &self, + &mut self, flash: &mut S, flash_range: Range, page_index: usize, @@ -99,3 +74,173 @@ impl StateQuery for NoCache { } } } + +pub struct NoCache; + +impl StateQuery for NoCache { + fn invalidate_cache_state(&mut self) {} + + fn mark_dirty(&mut self) {} + + fn unmark_dirty(&mut self) {} + + fn is_dirty(&self) -> bool { + false + } +} + +struct DirtyCache { + dirty: bool, +} + +impl StateQuery for DirtyCache { + fn invalidate_cache_state(&mut self) { + self.dirty = false; + } + + fn mark_dirty(&mut self) { + self.dirty = true; + } + + fn unmark_dirty(&mut self) { + self.dirty = false; + } + + fn is_dirty(&self) -> bool { + self.dirty + } +} + +pub struct PageStateCache { + dirty_cache: DirtyCache, + pages: [Option; PAGE_COUNT], +} + +impl PageStateCache { + pub const fn new() -> Self { + Self { + dirty_cache: DirtyCache { dirty: false }, + pages: [None; PAGE_COUNT], + } + } +} + +impl Default for PageStateCache { + fn default() -> Self { + Self::new() + } +} + +impl StateQuery for PageStateCache { + fn invalidate_cache_state(&mut self) { + self.dirty_cache.invalidate_cache_state(); + self.pages = [None; PAGE_COUNT]; + } + + fn mark_dirty(&mut self) { + self.dirty_cache.mark_dirty(); + } + + fn unmark_dirty(&mut self) { + self.dirty_cache.unmark_dirty(); + } + + fn is_dirty(&self) -> bool { + self.dirty_cache.is_dirty() + } + + fn notice_page_state(&mut self, page_index: usize, new_state: PageState) { + self.mark_dirty(); + self.pages[page_index] = Some(new_state); + } + + async fn get_page_state( + &mut self, + flash: &mut S, + flash_range: Range, + page_index: usize, + ) -> Result> { + match self.pages[page_index] { + Some(state) => Ok(state), + None => { + let state = NoCache + .get_page_state(flash, flash_range, page_index) + .await?; + self.pages[page_index] = Some(state); + Ok(state) + } + } + } +} + +#[cfg(test)] +mod queue_tests { + use crate::{ + mock_flash::{self, WriteCountCheck}, + queue::{peek, pop, push}, + }; + + use super::*; + use futures_test::test; + + const NUM_PAGES: usize = 4; + const LOOP_COUNT: usize = 2000; + + #[test] + async fn no_cache() { + assert_eq!(run_test(NoCache).await, (594934, 6299, 146)); + } + + #[test] + async fn page_state_cache() { + assert_eq!( + run_test(PageStateCache::::new()).await, + (308740, 6299, 146) + ); + } + + async fn run_test(mut cache: impl Cache) -> (u32, u32, u32) { + let mut flash = + mock_flash::MockFlashBase::::new(WriteCountCheck::Twice, None); + const FLASH_RANGE: Range = 0x00..0x400; + let mut data_buffer = [0; 1024]; + + for i in 0..LOOP_COUNT { + println!("{i}"); + let data = vec![i as u8; i % 20 + 1]; + + println!("PUSH"); + push(&mut flash, FLASH_RANGE, &mut cache, &data, true) + .await + .unwrap(); + assert_eq!( + &peek(&mut flash, FLASH_RANGE, &mut cache, &mut data_buffer) + .await + .unwrap() + .unwrap()[..], + &data, + "At {i}" + ); + println!("POP"); + assert_eq!( + &pop(&mut flash, FLASH_RANGE, &mut cache, &mut data_buffer) + .await + .unwrap() + .unwrap()[..], + &data, + "At {i}" + ); + println!("PEEK"); + assert_eq!( + peek(&mut flash, FLASH_RANGE, &mut cache, &mut data_buffer) + .await + .unwrap(), + None, + "At {i}" + ); + println!("DONE"); + } + + (flash.reads, flash.writes, flash.erases) + } +} diff --git a/src/lib.rs b/src/lib.rs index 8c5e846..44c561e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,7 @@ impl DerefMut for AlignedBuf { async fn try_general_repair( flash: &mut S, flash_range: Range, + query: &mut impl StateQuery, ) -> Result<(), Error> { // Loop through the pages and get their state. If one returns the corrupted error, // the page is likely half-erased. Fix for that is to re-erase again to hopefully finish the job. @@ -59,17 +60,7 @@ async fn try_general_repair( .await, Err(Error::Corrupted { .. }) ) { - flash - .erase( - calculate_page_address::(flash_range.clone(), page_index), - calculate_page_end_address::(flash_range.clone(), page_index), - ) - .await - .map_err(|e| Error::Storage { - value: e, - #[cfg(feature = "_test")] - backtrace: std::backtrace::Backtrace::capture(), - })?; + open_page(flash, flash_range.clone(), query, page_index).await?; } } @@ -147,6 +138,30 @@ const fn calculate_page_index(flash_range: Range, address: u32 /// The marker being used for page states const MARKER: u8 = 0; +/// Erase the page to open it again +async fn open_page( + flash: &mut S, + flash_range: Range, + query: &mut impl StateQuery, + page_index: usize, +) -> Result<(), Error> { + query.notice_page_state(page_index, PageState::Open); + + flash + .erase( + calculate_page_address::(flash_range.clone(), page_index), + calculate_page_end_address::(flash_range.clone(), page_index), + ) + .await + .map_err(|e| Error::Storage { + value: e, + #[cfg(feature = "_test")] + backtrace: std::backtrace::Backtrace::capture(), + })?; + + Ok(()) +} + /// Fully closes a page by writing both the start and end marker async fn close_page( flash: &mut S, @@ -161,6 +176,8 @@ async fn close_page( return Ok(()); } + query.notice_page_state(page_index, PageState::Closed); + let buffer = AlignedBuf([MARKER; MAX_WORD_SIZE]); // Close the end marker flash @@ -193,6 +210,14 @@ async fn partial_close_page( return Ok(current_state); } + let new_state = match current_state { + PageState::Closed => PageState::Closed, + PageState::PartialOpen => PageState::PartialOpen, + PageState::Open => PageState::PartialOpen, + }; + + query.notice_page_state(page_index, new_state); + let buffer = AlignedBuf([MARKER; MAX_WORD_SIZE]); // Close the start marker flash @@ -207,11 +232,7 @@ async fn partial_close_page( backtrace: std::backtrace::Backtrace::capture(), })?; - Ok(match current_state { - PageState::Closed => PageState::Closed, - PageState::PartialOpen => PageState::PartialOpen, - PageState::Open => PageState::PartialOpen, - }) + Ok(new_state) } /// The state of a page diff --git a/src/map.rs b/src/map.rs index 2a623f0..8d2519c 100644 --- a/src/map.rs +++ b/src/map.rs @@ -577,17 +577,7 @@ async fn migrate_items( } } - flash - .erase( - calculate_page_address::(flash_range.clone(), source_page), - calculate_page_end_address::(flash_range.clone(), source_page), - ) - .await - .map_err(|e| MapError::Storage { - value: e, - #[cfg(feature = "_test")] - backtrace: std::backtrace::Backtrace::capture(), - })?; + open_page(flash, flash_range.clone(), query, source_page).await?; Ok(()) } @@ -610,7 +600,7 @@ pub async fn try_repair( ) -> Result<(), MapError> { query.invalidate_cache_state(); - crate::try_general_repair(flash, flash_range.clone()).await?; + crate::try_general_repair(flash, flash_range.clone(), query).await?; // Let's check if we corrupted in the middle of a migration if let Some(partial_open_page) = find_first_page( @@ -630,17 +620,7 @@ pub async fn try_repair( { // Yes, the migration got interrupted. Let's redo it. // To do that, we erase the partial open page first because it contains incomplete data. - flash - .erase( - calculate_page_address::(flash_range.clone(), partial_open_page), - calculate_page_end_address::(flash_range.clone(), partial_open_page), - ) - .await - .map_err(|e| MapError::Storage { - value: e, - #[cfg(feature = "_test")] - backtrace: std::backtrace::Backtrace::capture(), - })?; + open_page(flash, flash_range.clone(), query, partial_open_page).await?; // Then partially close it again partial_close_page(flash, flash_range.clone(), &mut NoCache, partial_open_page).await?; diff --git a/src/queue.rs b/src/queue.rs index 1e3b6dd..fe84a8e 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -140,18 +140,7 @@ pub async fn push( return Err(Error::FullStorage); } - flash - .erase( - calculate_page_address::(flash_range.clone(), next_page), - calculate_page_end_address::(flash_range.clone(), next_page), - ) - .await - .map_err(|e| Error::Storage { - value: e, - #[cfg(feature = "_test")] - backtrace: std::backtrace::Backtrace::capture(), - })?; - + open_page(flash, flash_range.clone(), query, next_page).await?; close_page(flash, flash_range.clone(), query, current_page).await?; partial_close_page(flash, flash_range.clone(), query, next_page).await?; next_address = Some(next_page_data_start_address); @@ -568,8 +557,9 @@ async fn find_oldest_page( pub async fn try_repair( flash: &mut S, flash_range: Range, + query: &mut impl Cache, ) -> Result<(), Error> { - crate::try_general_repair(flash, flash_range.clone()).await?; + crate::try_general_repair(flash, flash_range.clone(), query).await?; Ok(()) }