From 299914731a5f1fb6661a92cb3647a802220b4c36 Mon Sep 17 00:00:00 2001 From: gferraro Date: Tue, 13 Aug 2024 14:30:20 +1200 Subject: [PATCH 01/12] offload last first --- src/core1_sub_tasks.rs | 2 +- src/onboard_flash.rs | 48 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index dc73d3e..7aeb1b8 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -147,7 +147,7 @@ pub fn offload_flash_storage_and_events( ); // do some offloading. let mut file_count = 0; - flash_storage.begin_offload(); + flash_storage.begin_offload_reverse(); let mut file_start = true; let mut part_count = 0; let mut success: bool = true; diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index e2757e4..ee11ce6 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -206,6 +206,10 @@ impl Page { self.user_metadata_1()[0] == 0 } + fn file_start(&self) -> u16 { + LittleEndian::read_u16(&self.user_metadata_1()[12..=13]) + } + fn is_last_page_for_file(&self) -> bool { self.user_metadata_1()[1] == 0 } @@ -296,8 +300,8 @@ pub struct OnboardFlash { dma_channel_2: Option>, record_to_flash: bool, pub payload_buffer: Option<&'static mut [u8; 2115]>, + file_start: Option, } - /// Each block is made up 64 pages of 2176 bytes. 139,264 bytes per block. /// Each page has a 2048 byte data storage section and a 128byte spare area for ECC codes. /// We should check for ECC errors after read/writes ourselves @@ -334,6 +338,7 @@ impl OnboardFlash { dma_channel_2: Some(dma_channel_2), record_to_flash: should_record, payload_buffer: payload_buffer, + file_start: None, } } pub fn init(&mut self) { @@ -696,6 +701,40 @@ impl OnboardFlash { } } + pub fn begin_offload_reverse(&mut self) { + if let Some(last_block_index) = self.last_used_block_index { + for block_index in (0..=last_block_index).rev() { + self.current_block_index = block_index; + if let Ok(page) = self.last_dirty_page(block_index) { + if self.current_page.is_last_page_for_file() { + let start_block = self.current_page.file_start(); + if start_block == u16::MAX { + //not written do something + } else { + self.current_block_index = start_block as isize + } + } else { + //shouldnt happen maybe corrupt file + //should keep cycling back + } + } + } + self.current_page_index = 0; + } + } + + fn last_dirty_page(&mut self, block_index: isize) -> Result { + for page in (0..64).rev() { + self.read_page(block_index, page).unwrap(); + self.read_page_metadata(0); + self.wait_for_all_ready(); + if self.current_page.page_is_used() { + return Ok(page); + } + } + Err(()) + } + pub fn read_from_cache_at_column_offset( &mut self, block: isize, @@ -809,8 +848,10 @@ impl OnboardFlash { "Starting file at file block {}, page {}", self.current_block_index, self.current_page_index ); + self.file_start = Some(self.current_block_index as u16); self.current_block_index } + pub fn finish_transfer( &mut self, block_index: Option, @@ -1060,6 +1101,11 @@ impl OnboardFlash { let space = &mut bytes[4..][0x820..=0x83f][10..=11]; LittleEndian::write_u16(space, crc); } + //need to check datasheet again + if is_last { + let space = &mut bytes[4..][0x820..=0x83f][12..=13]; + LittleEndian::write_u16(space, self.file_start.unwrap()); + } //info!("Wrote user meta {:?}", bytes[4..][0x820..=0x83f][0..10]); } // info!( From 4fc4ef687a76b311319fafb0071b16675f6ed74f Mon Sep 17 00:00:00 2001 From: gferraro Date: Thu, 15 Aug 2024 15:38:42 +1200 Subject: [PATCH 02/12] offload last first --- src/core0_audio.rs | 6 +- src/core0_task.rs | 3 +- src/core1_sub_tasks.rs | 161 ++++++++++++++++++++++++++++- src/core1_task.rs | 58 ++++++----- src/cptv_encoder/streaming_cptv.rs | 16 +-- src/onboard_flash.rs | 159 ++++++++++++++++++++-------- src/pdm_microphone.rs | 8 +- 7 files changed, 319 insertions(+), 92 deletions(-) diff --git a/src/core0_audio.rs b/src/core0_audio.rs index acb7951..de46b6c 100644 --- a/src/core0_audio.rs +++ b/src/core0_audio.rs @@ -661,7 +661,6 @@ fn should_offload_audio_recordings( if !has_files { return false; } - // flash getting full if flash_storage.is_too_full_for_audio() { info!("Offloading as flash is nearly full"); @@ -673,6 +672,11 @@ fn should_offload_audio_recordings( info!("Offloading as logger is nearly full"); return true; } + if flash_storage.file_start.is_none() { + //one off + info!("Offloading as previous file system version"); + return true; + } return false; } diff --git a/src/core0_task.rs b/src/core0_task.rs index a80ad96..7e5e6b9 100644 --- a/src/core0_task.rs +++ b/src/core0_task.rs @@ -488,7 +488,8 @@ pub fn frame_acquisition_loop( transferring_prev_frame = false; prev_frame_needs_transfer = false; } else if message == Core1Task::RequestReset.into() { - watchdog.start(100.micros()); + watchdog.start(8000.micros()); + info!("GOT RESET"); loop { // Wait until the watchdog timer kills us. nop(); diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index 7aeb1b8..8dd4f52 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -145,9 +145,162 @@ pub fn offload_flash_storage_and_events( None }, ); + let mut has_file; + let mut can_do_reverse; + (has_file, can_do_reverse) = flash_storage.begin_offload_reverse(); + if !can_do_reverse { + //old file system will do this once + return offload_flash_storage_forward( + flash_storage, + pi_spi, + resets, + dma, + clock_freq, + shared_i2c, + delay, + timer, + event_logger, + time, + watchdog, + ); + } + // do some offloading. + let mut file_count = 0; + let mut success: bool = true; + let mut counter = timer.get_counter(); + // TODO: Could speed this up slightly using cache_random_read interleaving on flash storage. + // Probably doesn't matter though. + let previous_block = Some(10); + while has_file { + let mut file_start = true; + let mut part_count = 0; + let mut file_ended = false; + while let Some(((part, crc, block_index, page_index), is_last, spi)) = + flash_storage.get_file_part() + { + if watchdog.is_some() { + watchdog.as_mut().unwrap().feed(); + } + pi_spi.enable(spi, resets); + let transfer_type = if file_start && !is_last { + ExtTransferMessage::BeginFileTransfer + } else if !file_start && !is_last { + ExtTransferMessage::ResumeFileTransfer + } else if is_last { + ExtTransferMessage::EndFileTransfer + } else if file_start && is_last { + ExtTransferMessage::BeginAndEndFileTransfer + } else { + crate::unreachable!("Invalid file transfer state"); + }; + + if file_start { + info!("Offload start is {}:{}", block_index, page_index) + } + if is_last { + info!("Got last file {}:{}", block_index, page_index); + } + let crc_check = Crc::::new(&CRC_16_XMODEM); + let current_crc = crc_check.checksum(&part); + if current_crc != crc { + // warn!( + // "Data corrupted at part #{} ({}:{}) in transfer to or from flash memory", + // part_count, block_index, page_index + // ); + } + + let mut attempts = 0; + 'transfer_part: loop { + if watchdog.is_some() { + watchdog.as_mut().unwrap().feed(); + } + let did_transfer = + pi_spi.send_message(transfer_type, &part, current_crc, dma, timer, resets); + counter = timer.get_counter(); + if !did_transfer { + attempts += 1; + if attempts > 100 { + success = false; + break 'transfer_part; + } + //takes tc2-agent about this long to poll again will fail a lot otherwise + let time_since = (timer.get_counter() - counter).to_micros(); + if time_since < TIME_BETWEEN_TRANSFER { + delay.delay_us((TIME_BETWEEN_TRANSFER - time_since) as u32); + } + } else { + break 'transfer_part; + } + } + + // Give spi peripheral back to flash storage. + if let Some(spi) = pi_spi.disable() { + flash_storage.take_spi(spi, resets, clock_freq.Hz()); + if is_last { + event_logger.log_event( + LoggerEvent::new( + LoggerEventKind::OffloadedRecording, + time.get_timestamp_micros(&timer), + ), + flash_storage, + ); + } + } + if !success { + break; + } + + part_count += 1; + if is_last { + file_count += 1; + info!("Offloaded {} file(s)", file_count); + let _ = flash_storage.erase_last_file(); + file_ended = true; + } + file_start = false; + } + if !success { + break; + } + + if !file_ended { + info!("In complete file {}:0 erasing", flash_storage.file_start); + if let Err(e) = flash_storage.erase_last_file() { + info!("Nothing to erase?? {}", e); + } + } + (has_file, can_do_reverse) = flash_storage.begin_offload_reverse(); + delay.delay_ms(1000); + } + if success { + info!( + "Completed file offload, transferred {} files start {} previous is {}", + file_count, flash_storage.file_start, flash_storage.previous_file_start + ); + file_count != 0 + } else { + flash_storage.scan(); + warn!("File transfer to pi failed"); + false + } +} + +pub fn offload_flash_storage_forward( + flash_storage: &mut OnboardFlash, + pi_spi: &mut ExtSpiTransfers, + resets: &mut RESETS, + dma: &mut DMA, + clock_freq: u32, + shared_i2c: &mut SharedI2C, + delay: &mut Delay, + timer: &mut Timer, + event_logger: &mut EventLogger, + time: &SyncedDateTime, + mut watchdog: Option<&mut bsp::hal::Watchdog>, +) -> bool { // do some offloading. let mut file_count = 0; - flash_storage.begin_offload_reverse(); + flash_storage.begin_offload(); let mut file_start = true; let mut part_count = 0; let mut success: bool = true; @@ -174,6 +327,12 @@ pub fn offload_flash_storage_and_events( crate::unreachable!("Invalid file transfer state"); }; + if file_start { + info!("Offload start is {}:{}", block_index, page_index) + } + // if is_last { + // info!("Got last file {}:{}", block_index, page_index); + // } let crc_check = Crc::::new(&CRC_16_XMODEM); let current_crc = crc_check.checksum(&part); if current_crc != crc { diff --git a/src/core1_task.rs b/src/core1_task.rs index d664771..51970fa 100644 --- a/src/core1_task.rs +++ b/src/core1_task.rs @@ -400,7 +400,9 @@ pub fn core_1_task( let has_files_to_offload = flash_storage.has_files_to_offload(); let should_offload = (has_files_to_offload && !device_config.time_is_in_recording_window(&synced_date_time.date_time_utc, &None)) - || flash_storage.is_too_full_to_start_new_recordings(); + || flash_storage.is_too_full_to_start_new_recordings() + || (has_files_to_offload && flash_storage.file_start.is_none()); + //means old file system offload once let should_offload = if !should_offload { let previous_offload_time = event_logger .latest_event_of_kind(LoggerEventKind::OffloadedRecording, &mut flash_storage) @@ -419,7 +421,6 @@ pub fn core_1_task( } else { should_offload }; - let did_offload_files = if should_offload { offload_flash_storage_and_events( &mut flash_storage, @@ -437,7 +438,6 @@ pub fn core_1_task( } else { false }; - // Unset the is_recording flag on attiny on startup let _ = shared_i2c .set_recording_flag(&mut delay, false) @@ -485,12 +485,15 @@ pub fn core_1_task( let mut logged_flash_storage_nearly_full = false; // NOTE: If there are already recordings on the flash memory, // assume we've already made the startup status recording during this recording window. - let mut made_startup_status_recording = !has_files_to_offload || did_offload_files; + let mut made_startup_status_recording = has_files_to_offload || did_offload_files; let mut made_shutdown_status_recording = false; let mut making_status_recording = false; // Enable raw frame transfers to pi – if not already enabled. pi_spi.enable_pio_spi(); - info!("Entering frame loop"); + info!( + "Entering frame loop {} has files {} did offload {}", + made_startup_status_recording, has_files_to_offload, did_offload_files + ); loop { let input = sio.fifo.read_blocking(); crate::assert_eq!( @@ -759,10 +762,32 @@ pub fn core_1_task( // Finalise on a different frame period to writing out the prev/last frame, // to give more breathing room. if let Some(cptv_stream) = &mut cptv_stream { - error!("Ending current recording"); let cptv_start_block_index = cptv_stream.starting_block_index as isize; - cptv_stream.finalise(&mut flash_storage); let cptv_end_block_index = flash_storage.last_used_block_index.unwrap(); + error!( + "Ending current recording start block {} end block{}", + cptv_start_block_index, cptv_end_block_index + ); + + if !making_status_recording + && motion_detection.as_ref().unwrap().was_false_positive() + // && cptv_stream.num_frames <= 100 + { + info!( + "Discarding as a false-positive {}:{} ", + cptv_start_block_index, cptv_end_block_index + ); + let _ = flash_storage.erase_last_file(); + event_logger.log_event( + LoggerEvent::new( + LoggerEventKind::WouldDiscardAsFalsePositive, + synced_date_time.get_timestamp_micros(&timer), + ), + &mut flash_storage, + ); + } else { + cptv_stream.finalise(&mut flash_storage); + } ended_recording = true; let _ = shared_i2c @@ -776,24 +801,7 @@ pub fn core_1_task( ), &mut flash_storage, ); - if !making_status_recording - && motion_detection.as_ref().unwrap().was_false_positive() - // && cptv_stream.num_frames <= 100 - { - info!("Discarding as a false-positive"); - cptv_stream.discard( - &mut flash_storage, - cptv_start_block_index, - cptv_end_block_index, - ); - // event_logger.log_event( - // LoggerEvent::new( - // LoggerEventKind::WouldDiscardAsFalsePositive, - // synced_date_time.get_timestamp_micros(&timer), - // ), - // &mut flash_storage, - // ); - } + if making_status_recording { making_status_recording = false; } diff --git a/src/cptv_encoder/streaming_cptv.rs b/src/cptv_encoder/streaming_cptv.rs index 3218d52..33f59d2 100644 --- a/src/cptv_encoder/streaming_cptv.rs +++ b/src/cptv_encoder/streaming_cptv.rs @@ -557,7 +557,7 @@ impl<'a> CptvStream<'a> { for b in buf { self.cursor.write_byte(b); if let Some((to_flush, num_bytes)) = self.cursor.should_flush() { - flash_storage.append_file_bytes( + let _ = flash_storage.append_file_bytes( to_flush, num_bytes, false, @@ -572,7 +572,7 @@ impl<'a> CptvStream<'a> { for b in buf { self.cursor.write_byte(b); if let Some((to_flush, num_bytes)) = self.cursor.should_flush() { - flash_storage.append_file_bytes( + let _ = flash_storage.append_file_bytes( to_flush, num_bytes, false, @@ -586,7 +586,7 @@ impl<'a> CptvStream<'a> { // and write out to storage. let _ = self.cursor.end_aligned(); let (to_flush, num_bytes) = self.cursor.flush(); - flash_storage.append_file_bytes( + let _ = flash_storage.append_file_bytes( to_flush, num_bytes, !at_header_location, @@ -621,16 +621,6 @@ impl<'a> CptvStream<'a> { } self.write_gzip_trailer(flash_storage, true); } - - pub fn discard( - &mut self, - flash_storage: &mut OnboardFlash, - start_block_index: isize, - end_block_index: isize, - ) { - // NOTE: In the case that the block erase fails, that just means we won't reclaim the space at the moment. - let _ = flash_storage.erase_block_range(start_block_index, end_block_index); - } } struct HeaderIterator { diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index ee11ce6..b200931 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -14,6 +14,7 @@ use crate::bsp::pac::SPI1; use crate::rp2040_flash::PAGE_SIZE; use byteorder::{ByteOrder, LittleEndian}; +use core::char::MAX; use core::mem; use cortex_m::singleton; use crc::{Crc, CRC_16_XMODEM}; @@ -206,8 +207,22 @@ impl Page { self.user_metadata_1()[0] == 0 } - fn file_start(&self) -> u16 { - LittleEndian::read_u16(&self.user_metadata_1()[12..=13]) + pub fn file_start(&self) -> Option { + let start = LittleEndian::read_u16(&self.user_metadata_1()[12..=13]); + if start == u16::MAX { + None + } else { + Some(start) + } + } + + pub fn previous_file_start(&self) -> Option<(u16)> { + let block = LittleEndian::read_u16(&self.user_metadata_1()[14..=15]); + if block == u16::MAX { + None + } else { + Some(block) + } } fn is_last_page_for_file(&self) -> bool { @@ -292,6 +307,7 @@ pub struct OnboardFlash { pub current_page_index: isize, pub current_block_index: isize, pub last_used_block_index: Option, + pub previous_file_start: Option, pub first_used_block_index: Option, pub bad_blocks: [i16; 40], pub current_page: Page, @@ -300,7 +316,7 @@ pub struct OnboardFlash { dma_channel_2: Option>, record_to_flash: bool, pub payload_buffer: Option<&'static mut [u8; 2115]>, - file_start: Option, + pub file_start: Option, } /// Each block is made up 64 pages of 2176 bytes. 139,264 bytes per block. /// Each page has a 2048 byte data storage section and a 128byte spare area for ECC codes. @@ -329,6 +345,7 @@ impl OnboardFlash { spi: None, first_used_block_index: None, last_used_block_index: None, + // previous_file_index: None, bad_blocks: [i16::MAX; 40], current_page_index: 0, current_block_index: 0, @@ -339,6 +356,7 @@ impl OnboardFlash { record_to_flash: should_record, payload_buffer: payload_buffer, file_start: None, + previous_file_start: None, } } pub fn init(&mut self) { @@ -385,7 +403,7 @@ impl OnboardFlash { // TODO: Interleave with random cache read // TODO: We can see if this is faster if we just read the column index of the end of the page? // For simplicity at the moment, just read the full pages - self.read_page(block_index, 0).unwrap(); + self.read_page(block_index, 1).unwrap(); self.read_page_metadata(block_index); self.wait_for_all_ready(); if self.current_page.is_part_of_bad_block() { @@ -403,9 +421,14 @@ impl OnboardFlash { if self.last_used_block_index.is_none() && self.first_used_block_index.is_some() { self.last_used_block_index = Some(block_index - 1); + self.file_start = self.prev_page.file_start(); + self.current_block_index = block_index; self.current_page_index = 0; - println!("Setting next starting block index {}", block_index); + println!( + "Setting next starting block index {} last used is {}", + block_index, self.last_used_block_index + ); } } else { let address = OnboardFlash::get_address(block_index, 0); @@ -515,6 +538,7 @@ impl OnboardFlash { } self.first_used_block_index = None; self.last_used_block_index = None; + self.file_start = None; self.current_page_index = 0; self.current_block_index = 0; } @@ -533,12 +557,28 @@ impl OnboardFlash { } } - pub fn erase_block_range( - &mut self, - start_block_index: isize, - end_block_index: isize, - ) -> Result<(), &str> { - for block_index in start_block_index..=end_block_index { + pub fn erase_last_file(&mut self) -> Result<(), &str> { + //havent started a file + //haven't used a block + //havent written to file yet + + let (start, end) = (self.file_start, self.last_used_block_index); + + if start.is_none() + || self.last_used_block_index.is_none() + || self.last_used_block_index.unwrap() < start.unwrap() as isize + { + // self.last_used_block_index = self.previous_file_index; + info!( + "Nothing to erase start {} last used block {}", + start, self.last_used_block_index + ); + return Err("File hasn't been written too"); + } + let start_block_index = start.unwrap() as isize; + info!("Erasing last file {}:0 to {}", start_block_index, end); + + for block_index in start_block_index..=end.unwrap() { if self.bad_blocks.contains(&(block_index as i16)) { info!("Skipping erase of bad block {}", block_index); } else if !self.erase_block(block_index).is_ok() { @@ -546,6 +586,9 @@ impl OnboardFlash { return Err("Block erase failed"); } } + self.file_start = self.previous_file_start; + self.previous_file_start = None; + info!("Set file start {}", self.file_start); if start_block_index == 0 { self.first_used_block_index = None; self.last_used_block_index = None; @@ -584,15 +627,21 @@ impl OnboardFlash { let length = self.current_page.page_bytes_used(); let crc = self.current_page.page_crc(); let is_last_page_for_file = self.current_page.is_last_page_for_file(); + if is_last_page_for_file { + info!( + "Got last file {}:{} file start is {} previous file start is {}", + self.current_block_index, + self.current_page_index, + self.current_page.file_start(), + self.current_page.previous_file_start() + ); + } + let block = self.current_block_index; + let page = self.current_page_index; self.advance_file_cursor(is_last_page_for_file); let spi = self.free_spi().unwrap(); Some(( - ( - &self.current_page.user_data()[0..length], - crc, - self.current_block_index, - self.current_page_index, - ), + (&self.current_page.user_data()[0..length], crc, block, page), is_last_page_for_file, spi, )) @@ -671,10 +720,10 @@ impl OnboardFlash { // that the file spans as temporarily corrupt, so this file doesn't get read and // send to the raspberry pi //Err(&"unrecoverable data corruption error") - warn!( - "unrecoverable data corruption error at {}:{} - should relocate? {}", - block, page, should_relocate - ); + // warn!( + // "unrecoverable data corruption error at {}:{} - should relocate? {}", + // block, page, should_relocate + // ); Ok(()) } } @@ -701,32 +750,43 @@ impl OnboardFlash { } } - pub fn begin_offload_reverse(&mut self) { + pub fn begin_offload_reverse(&mut self) -> (bool, bool) { if let Some(last_block_index) = self.last_used_block_index { - for block_index in (0..=last_block_index).rev() { - self.current_block_index = block_index; - if let Ok(page) = self.last_dirty_page(block_index) { - if self.current_page.is_last_page_for_file() { - let start_block = self.current_page.file_start(); - if start_block == u16::MAX { - //not written do something - } else { - self.current_block_index = start_block as isize - } - } else { - //shouldnt happen maybe corrupt file - //should keep cycling back - } + info!("Offload reverse last used {}", self.last_used_block_index); + self.read_page(last_block_index, 0).unwrap(); + self.read_page_metadata(last_block_index); + self.wait_for_all_ready(); + if self.current_page.page_is_used() { + info!("Page used {}:{}", last_block_index, 0); + if let Some(start_block) = self.current_page.file_start() { + self.current_block_index = start_block as isize; + self.current_page_index = 0; + self.file_start = Some(self.current_block_index as u16); + self.previous_file_start = self.current_page.previous_file_start(); + info!( + "Set file start to {}:{} and previous {}", + self.current_block_index, 0, self.previous_file_start + ); + return (true, true); + } else { + //do something old file system only need to offload them once + return (true, false); } + } else { + //this shouldn't happen either + warn!("Last used block is empty {}:{}", last_block_index, 0); + return (false, true); } - self.current_page_index = 0; + } else { + (false, true) } } fn last_dirty_page(&mut self, block_index: isize) -> Result { for page in (0..64).rev() { + info!("Looking for dirty page at {}:{}", block_index, page); self.read_page(block_index, page).unwrap(); - self.read_page_metadata(0); + self.read_page_metadata(block_index); self.wait_for_all_ready(); if self.current_page.page_is_used() { return Ok(page); @@ -841,12 +901,13 @@ impl OnboardFlash { info!("Device id {:x}", id); } pub fn start_file(&mut self, start_page: isize) -> isize { - // We always start writing a new file at page 1, reserving page 0 for when we come back + // CPTV always start writing a new file at page 1, reserving page 0 for when we come back // and write the header once we've finished writing the file. + self.previous_file_start = self.file_start; self.current_page_index = start_page; warn!( - "Starting file at file block {}, page {}", - self.current_block_index, self.current_page_index + "Starting file at file block {}, page {} previous is {}", + self.current_block_index, self.current_page_index, self.previous_file_start ); self.file_start = Some(self.current_block_index as u16); self.current_block_index @@ -895,6 +956,7 @@ impl OnboardFlash { self.last_used_block_index = Some(b); } } + if block_index.is_none() && page_index.is_none() { self.advance_file_cursor(is_last); } @@ -1101,10 +1163,14 @@ impl OnboardFlash { let space = &mut bytes[4..][0x820..=0x83f][10..=11]; LittleEndian::write_u16(space, crc); } - //need to check datasheet again - if is_last { - let space = &mut bytes[4..][0x820..=0x83f][12..=13]; - LittleEndian::write_u16(space, self.file_start.unwrap()); + //write file start block + let space = &mut bytes[4..][0x820..=0x83f][12..=13]; + LittleEndian::write_u16(space, self.file_start.unwrap()); + if self.file_start.unwrap() > 0 { + // if let Some(previous_start_block) = self.previous_file_start { + //write previous file start could just write on first page if it matters + let space = &mut bytes[4..][0x820..=0x83f][14..=15]; + LittleEndian::write_u16(space, self.previous_file_start.unwrap() as u16); } //info!("Wrote user meta {:?}", bytes[4..][0x820..=0x83f][0..10]); } @@ -1127,6 +1193,9 @@ impl OnboardFlash { // TODO: Check ECC status, mark and relocate block if needed. //info!("Status after program {:#010b}", status.inner); if !status.program_failed() { + if self.first_used_block_index.is_none() { + self.first_used_block_index = Some(b); + } if self.first_used_block_index.is_none() { self.first_used_block_index = Some(b); } diff --git a/src/pdm_microphone.rs b/src/pdm_microphone.rs index 6d98dd0..66852a5 100644 --- a/src/pdm_microphone.rs +++ b/src/pdm_microphone.rs @@ -296,12 +296,8 @@ impl PdmMicrophone { true, ); } - if flash_storage.last_used_block_index.is_some() { - flash_storage.erase_block_range( - start_block_index, - flash_storage.last_used_block_index.unwrap(), - ); - } + let _ = flash_storage.erase_last_file(); + break; } // When a transfer is done we immediately enqueue the buffers again. From 89458c7715e66bf36dbfef768ce04b085a500949 Mon Sep 17 00:00:00 2001 From: gferraro Date: Thu, 15 Aug 2024 20:08:04 +1200 Subject: [PATCH 03/12] offload file --- src/core0_task.rs | 3 +- src/core1_sub_tasks.rs | 49 ++++++++++++------------------- src/event_logger.rs | 3 ++ src/onboard_flash.rs | 66 +++++++++++++++++++++++------------------- 4 files changed, 60 insertions(+), 61 deletions(-) diff --git a/src/core0_task.rs b/src/core0_task.rs index 7e5e6b9..a80ad96 100644 --- a/src/core0_task.rs +++ b/src/core0_task.rs @@ -488,8 +488,7 @@ pub fn frame_acquisition_loop( transferring_prev_frame = false; prev_frame_needs_transfer = false; } else if message == Core1Task::RequestReset.into() { - watchdog.start(8000.micros()); - info!("GOT RESET"); + watchdog.start(100.micros()); loop { // Wait until the watchdog timer kills us. nop(); diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index 8dd4f52..9e24eaa 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -145,25 +145,8 @@ pub fn offload_flash_storage_and_events( None }, ); - let mut has_file; - let mut can_do_reverse; - (has_file, can_do_reverse) = flash_storage.begin_offload_reverse(); - if !can_do_reverse { - //old file system will do this once - return offload_flash_storage_forward( - flash_storage, - pi_spi, - resets, - dma, - clock_freq, - shared_i2c, - delay, - timer, - event_logger, - time, - watchdog, - ); - } + let mut has_file = flash_storage.begin_offload_reverse(); + // do some offloading. let mut file_count = 0; let mut success: bool = true; @@ -203,10 +186,10 @@ pub fn offload_flash_storage_and_events( let crc_check = Crc::::new(&CRC_16_XMODEM); let current_crc = crc_check.checksum(&part); if current_crc != crc { - // warn!( - // "Data corrupted at part #{} ({}:{}) in transfer to or from flash memory", - // part_count, block_index, page_index - // ); + warn!( + "Data corrupted at part #{} ({}:{}) in transfer to or from flash memory", + part_count, block_index, page_index + ); } let mut attempts = 0; @@ -264,13 +247,21 @@ pub fn offload_flash_storage_and_events( } if !file_ended { - info!("In complete file {}:0 erasing", flash_storage.file_start); + info!( + "Incomplete file at block {} erasing", + flash_storage.file_start + ); if let Err(e) = flash_storage.erase_last_file() { - info!("Nothing to erase?? {}", e); + event_logger.log_event( + LoggerEvent::new( + LoggerEventKind::EmptyErase, + time.get_timestamp_micros(&timer), + ), + flash_storage, + ); } } - (has_file, can_do_reverse) = flash_storage.begin_offload_reverse(); - delay.delay_ms(1000); + has_file = flash_storage.begin_offload_reverse(); } if success { info!( @@ -285,6 +276,7 @@ pub fn offload_flash_storage_and_events( } } +//old code just used when moving old files pub fn offload_flash_storage_forward( flash_storage: &mut OnboardFlash, pi_spi: &mut ExtSpiTransfers, @@ -330,9 +322,6 @@ pub fn offload_flash_storage_forward( if file_start { info!("Offload start is {}:{}", block_index, page_index) } - // if is_last { - // info!("Got last file {}:{}", block_index, page_index); - // } let crc_check = Crc::::new(&CRC_16_XMODEM); let current_crc = crc_check.checksum(&part); if current_crc != crc { diff --git a/src/event_logger.rs b/src/event_logger.rs index ccfb120..c596db2 100644 --- a/src/event_logger.rs +++ b/src/event_logger.rs @@ -30,6 +30,7 @@ pub enum LoggerEventKind { AttinyCommError, Rp2040MissedAudioAlarm(u64), AudioRecordingFailed, + EmptyErase, } impl Into for LoggerEventKind { @@ -57,6 +58,7 @@ impl Into for LoggerEventKind { AttinyCommError => 19, Rp2040MissedAudioAlarm(_) => 20, AudioRecordingFailed => 21, + EmptyErase => 22, } } } @@ -88,6 +90,7 @@ impl TryFrom for LoggerEventKind { 19 => Ok(AttinyCommError), 20 => Ok(Rp2040MissedAudioAlarm(0)), 21 => Ok(AudioRecordingFailed), + 22 => Ok(EmptyErase), _ => Err(()), } } diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index b200931..a2cfa06 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -750,49 +750,57 @@ impl OnboardFlash { } } - pub fn begin_offload_reverse(&mut self) -> (bool, bool) { + pub fn begin_offload_reverse(&mut self) -> bool { if let Some(last_block_index) = self.last_used_block_index { - info!("Offload reverse last used {}", self.last_used_block_index); - self.read_page(last_block_index, 0).unwrap(); - self.read_page_metadata(last_block_index); - self.wait_for_all_ready(); - if self.current_page.page_is_used() { - info!("Page used {}:{}", last_block_index, 0); - if let Some(start_block) = self.current_page.file_start() { - self.current_block_index = start_block as isize; - self.current_page_index = 0; - self.file_start = Some(self.current_block_index as u16); - self.previous_file_start = self.current_page.previous_file_start(); - info!( - "Set file start to {}:{} and previous {}", - self.current_block_index, 0, self.previous_file_start - ); - return (true, true); - } else { - //do something old file system only need to offload them once - return (true, false); + if let Some(file_start) = self.file_start { + self.current_block_index = file_start as isize; + self.current_page_index = 0; + self.previous_file_start = self.current_page.previous_file_start(); + info!( + "Set file start to {}:{} and previous {}", + self.current_block_index, 0, self.previous_file_start + ); + return true; + } + //old file system should only happen once + self.current_block_index = self.find_start(last_block_index); + self.current_page_index = 0; + self.file_start = Some(self.current_block_index as u16); + info!( + "Searched for file start found {}:{}", + self.current_block_index, self.last_used_block_index + ); + return true; + } + return false; + } + + pub fn find_start(&mut self, file_block_index: isize) -> isize { + if file_block_index == 0 { + return 0; + } + //find previous last page for file + for block_index in (1..=file_block_index - 1).rev() { + if let Some(last_page) = self.last_dirty_page(block_index) { + if self.current_page.is_last_page_for_file() { + return block_index + 1; } - } else { - //this shouldn't happen either - warn!("Last used block is empty {}:{}", last_block_index, 0); - return (false, true); } - } else { - (false, true) } + return 0; } - fn last_dirty_page(&mut self, block_index: isize) -> Result { + fn last_dirty_page(&mut self, block_index: isize) -> Option { for page in (0..64).rev() { info!("Looking for dirty page at {}:{}", block_index, page); self.read_page(block_index, page).unwrap(); self.read_page_metadata(block_index); self.wait_for_all_ready(); if self.current_page.page_is_used() { - return Ok(page); + return Some(page); } } - Err(()) + None } pub fn read_from_cache_at_column_offset( From 243fe4163cdf6b27e243655a05397b09299ff790 Mon Sep 17 00:00:00 2001 From: gferraro Date: Fri, 16 Aug 2024 15:35:09 +1200 Subject: [PATCH 04/12] added more watchdog feed --- src/core1_sub_tasks.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index 9e24eaa..e83a14c 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -237,6 +237,9 @@ pub fn offload_flash_storage_and_events( if is_last { file_count += 1; info!("Offloaded {} file(s)", file_count); + if watchdog.is_some() { + watchdog.as_mut().unwrap().feed(); + } let _ = flash_storage.erase_last_file(); file_ended = true; } @@ -251,6 +254,9 @@ pub fn offload_flash_storage_and_events( "Incomplete file at block {} erasing", flash_storage.file_start ); + if watchdog.is_some() { + watchdog.as_mut().unwrap().feed(); + } if let Err(e) = flash_storage.erase_last_file() { event_logger.log_event( LoggerEvent::new( From 1d46dd98d3675e14ebba4c16e91d6c2fad5bc145 Mon Sep 17 00:00:00 2001 From: gferraro Date: Fri, 16 Aug 2024 15:48:19 +1200 Subject: [PATCH 05/12] tidy up --- src/core1_sub_tasks.rs | 125 ----------------------------------------- src/core1_task.rs | 16 ++---- src/onboard_flash.rs | 66 ++++++++++------------ 3 files changed, 35 insertions(+), 172 deletions(-) diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index e83a14c..00c143c 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -282,131 +282,6 @@ pub fn offload_flash_storage_and_events( } } -//old code just used when moving old files -pub fn offload_flash_storage_forward( - flash_storage: &mut OnboardFlash, - pi_spi: &mut ExtSpiTransfers, - resets: &mut RESETS, - dma: &mut DMA, - clock_freq: u32, - shared_i2c: &mut SharedI2C, - delay: &mut Delay, - timer: &mut Timer, - event_logger: &mut EventLogger, - time: &SyncedDateTime, - mut watchdog: Option<&mut bsp::hal::Watchdog>, -) -> bool { - // do some offloading. - let mut file_count = 0; - flash_storage.begin_offload(); - let mut file_start = true; - let mut part_count = 0; - let mut success: bool = true; - let mut counter = timer.get_counter(); - - // TODO: Could speed this up slightly using cache_random_read interleaving on flash storage. - // Probably doesn't matter though. - while let Some(((part, crc, block_index, page_index), is_last, spi)) = - flash_storage.get_file_part() - { - if watchdog.is_some() { - watchdog.as_mut().unwrap().feed(); - } - pi_spi.enable(spi, resets); - let transfer_type = if file_start && !is_last { - ExtTransferMessage::BeginFileTransfer - } else if !file_start && !is_last { - ExtTransferMessage::ResumeFileTransfer - } else if is_last { - ExtTransferMessage::EndFileTransfer - } else if file_start && is_last { - ExtTransferMessage::BeginAndEndFileTransfer - } else { - crate::unreachable!("Invalid file transfer state"); - }; - - if file_start { - info!("Offload start is {}:{}", block_index, page_index) - } - let crc_check = Crc::::new(&CRC_16_XMODEM); - let current_crc = crc_check.checksum(&part); - if current_crc != crc { - warn!( - "Data corrupted at part #{} ({}:{}) in transfer to or from flash memory", - part_count, block_index, page_index - ); - } - - let mut attempts = 0; - 'transfer_part: loop { - if watchdog.is_some() { - watchdog.as_mut().unwrap().feed(); - } - let did_transfer = - pi_spi.send_message(transfer_type, &part, current_crc, dma, timer, resets); - counter = timer.get_counter(); - if !did_transfer { - attempts += 1; - if attempts > 100 { - success = false; - break 'transfer_part; - } - //takes tc2-agent about this long to poll again will fail a lot otherwise - let time_since = (timer.get_counter() - counter).to_micros(); - if time_since < TIME_BETWEEN_TRANSFER { - delay.delay_us((TIME_BETWEEN_TRANSFER - time_since) as u32); - } - } else { - break 'transfer_part; - } - } - - // Give spi peripheral back to flash storage. - if let Some(spi) = pi_spi.disable() { - flash_storage.take_spi(spi, resets, clock_freq.Hz()); - if is_last { - event_logger.log_event( - LoggerEvent::new( - LoggerEventKind::OffloadedRecording, - time.get_timestamp_micros(&timer), - ), - flash_storage, - ); - } - } - if !success { - info!("NOT success so breaking"); - break; - } - - part_count += 1; - if is_last { - file_count += 1; - info!("Offloaded {} file(s)", file_count); - file_start = true; - } else { - file_start = false; - } - } - if success { - info!("Completed file offload, transferred {} files", file_count); - // TODO: Some validation from the raspberry pi that the transfer completed - // without errors, in the form of a hash, and if we have errors, we'd re-transmit. - - // Once we've successfully offloaded all files, we can erase the flash and we're - // ready to start recording new CPTV files there. - - info!("Erasing after successful offload"); - //flash_storage.erase_all_good_used_blocks(); - flash_storage.erase_all_blocks(); - file_count != 0 - } else { - flash_storage.scan(); - warn!("File transfer to pi failed"); - false - } -} - /// Returns `(Option, true)` when config was updated pub fn get_existing_device_config_or_config_from_pi_on_initial_handshake( flash_storage: &mut OnboardFlash, diff --git a/src/core1_task.rs b/src/core1_task.rs index 51970fa..df15ca3 100644 --- a/src/core1_task.rs +++ b/src/core1_task.rs @@ -490,10 +490,7 @@ pub fn core_1_task( let mut making_status_recording = false; // Enable raw frame transfers to pi – if not already enabled. pi_spi.enable_pio_spi(); - info!( - "Entering frame loop {} has files {} did offload {}", - made_startup_status_recording, has_files_to_offload, did_offload_files - ); + info!("Entering frame loop"); loop { let input = sio.fifo.read_blocking(); crate::assert_eq!( @@ -763,11 +760,6 @@ pub fn core_1_task( // to give more breathing room. if let Some(cptv_stream) = &mut cptv_stream { let cptv_start_block_index = cptv_stream.starting_block_index as isize; - let cptv_end_block_index = flash_storage.last_used_block_index.unwrap(); - error!( - "Ending current recording start block {} end block{}", - cptv_start_block_index, cptv_end_block_index - ); if !making_status_recording && motion_detection.as_ref().unwrap().was_false_positive() @@ -775,7 +767,7 @@ pub fn core_1_task( { info!( "Discarding as a false-positive {}:{} ", - cptv_start_block_index, cptv_end_block_index + cptv_start_block_index, flash_storage.last_used_block_index ); let _ = flash_storage.erase_last_file(); event_logger.log_event( @@ -787,6 +779,10 @@ pub fn core_1_task( ); } else { cptv_stream.finalise(&mut flash_storage); + error!( + "Ending current recording start block {} end block{}", + cptv_start_block_index, flash_storage.last_used_block_index + ); } ended_recording = true; diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index a2cfa06..de64ebf 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -345,7 +345,6 @@ impl OnboardFlash { spi: None, first_used_block_index: None, last_used_block_index: None, - // previous_file_index: None, bad_blocks: [i16::MAX; 40], current_page_index: 0, current_block_index: 0, @@ -403,6 +402,8 @@ impl OnboardFlash { // TODO: Interleave with random cache read // TODO: We can see if this is faster if we just read the column index of the end of the page? // For simplicity at the moment, just read the full pages + + //incase of incomplete cptv files page 0 will be empty self.read_page(block_index, 1).unwrap(); self.read_page_metadata(block_index); self.wait_for_all_ready(); @@ -425,10 +426,7 @@ impl OnboardFlash { self.current_block_index = block_index; self.current_page_index = 0; - println!( - "Setting next starting block index {} last used is {}", - block_index, self.last_used_block_index - ); + println!("Setting next starting block index {}", block_index); } } else { let address = OnboardFlash::get_address(block_index, 0); @@ -539,6 +537,7 @@ impl OnboardFlash { self.first_used_block_index = None; self.last_used_block_index = None; self.file_start = None; + self.previous_file_start = None; self.current_page_index = 0; self.current_block_index = 0; } @@ -562,23 +561,24 @@ impl OnboardFlash { //haven't used a block //havent written to file yet - let (start, end) = (self.file_start, self.last_used_block_index); - - if start.is_none() + if self.file_start.is_none() || self.last_used_block_index.is_none() - || self.last_used_block_index.unwrap() < start.unwrap() as isize + || self.last_used_block_index.unwrap() < self.file_start.unwrap() as isize { // self.last_used_block_index = self.previous_file_index; info!( "Nothing to erase start {} last used block {}", - start, self.last_used_block_index + self.file_start, self.last_used_block_index ); return Err("File hasn't been written too"); } - let start_block_index = start.unwrap() as isize; - info!("Erasing last file {}:0 to {}", start_block_index, end); + let start_block_index = self.file_start.unwrap() as isize; + info!( + "Erasing last file {}:0 to {}", + start_block_index, self.last_used_block_index + ); - for block_index in start_block_index..=end.unwrap() { + for block_index in start_block_index..=self.last_used_block_index.unwrap() { if self.bad_blocks.contains(&(block_index as i16)) { info!("Skipping erase of bad block {}", block_index); } else if !self.erase_block(block_index).is_ok() { @@ -588,7 +588,6 @@ impl OnboardFlash { } self.file_start = self.previous_file_start; self.previous_file_start = None; - info!("Set file start {}", self.file_start); if start_block_index == 0 { self.first_used_block_index = None; self.last_used_block_index = None; @@ -627,15 +626,15 @@ impl OnboardFlash { let length = self.current_page.page_bytes_used(); let crc = self.current_page.page_crc(); let is_last_page_for_file = self.current_page.is_last_page_for_file(); - if is_last_page_for_file { - info!( - "Got last file {}:{} file start is {} previous file start is {}", - self.current_block_index, - self.current_page_index, - self.current_page.file_start(), - self.current_page.previous_file_start() - ); - } + // if is_last_page_for_file { + // info!( + // "Got last file {}:{} file start is {} previous file start is {}", + // self.current_block_index, + // self.current_page_index, + // self.current_page.file_start(), + // self.current_page.previous_file_start() + // ); + // } let block = self.current_block_index; let page = self.current_page_index; self.advance_file_cursor(is_last_page_for_file); @@ -720,10 +719,10 @@ impl OnboardFlash { // that the file spans as temporarily corrupt, so this file doesn't get read and // send to the raspberry pi //Err(&"unrecoverable data corruption error") - // warn!( - // "unrecoverable data corruption error at {}:{} - should relocate? {}", - // block, page, should_relocate - // ); + warn!( + "unrecoverable data corruption error at {}:{} - should relocate? {}", + block, page, should_relocate + ); Ok(()) } } @@ -779,7 +778,7 @@ impl OnboardFlash { if file_block_index == 0 { return 0; } - //find previous last page for file + //find previous file end for block_index in (1..=file_block_index - 1).rev() { if let Some(last_page) = self.last_dirty_page(block_index) { if self.current_page.is_last_page_for_file() { @@ -1174,11 +1173,10 @@ impl OnboardFlash { //write file start block let space = &mut bytes[4..][0x820..=0x83f][12..=13]; LittleEndian::write_u16(space, self.file_start.unwrap()); - if self.file_start.unwrap() > 0 { - // if let Some(previous_start_block) = self.previous_file_start { + if let Some(previous_start) = self.previous_file_start { //write previous file start could just write on first page if it matters let space = &mut bytes[4..][0x820..=0x83f][14..=15]; - LittleEndian::write_u16(space, self.previous_file_start.unwrap() as u16); + LittleEndian::write_u16(space, previous_start as u16); } //info!("Wrote user meta {:?}", bytes[4..][0x820..=0x83f][0..10]); } @@ -1201,12 +1199,6 @@ impl OnboardFlash { // TODO: Check ECC status, mark and relocate block if needed. //info!("Status after program {:#010b}", status.inner); if !status.program_failed() { - if self.first_used_block_index.is_none() { - self.first_used_block_index = Some(b); - } - if self.first_used_block_index.is_none() { - self.first_used_block_index = Some(b); - } if self.last_used_block_index.is_none() { self.last_used_block_index = Some(b); } else if let Some(last_used_block_index) = self.last_used_block_index { From df3fdd23ef978d5c125dad17868384ebdebb873e Mon Sep 17 00:00:00 2001 From: gferraro Date: Fri, 16 Aug 2024 15:49:00 +1200 Subject: [PATCH 06/12] add first used again --- src/onboard_flash.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index de64ebf..cad40e8 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -1199,6 +1199,9 @@ impl OnboardFlash { // TODO: Check ECC status, mark and relocate block if needed. //info!("Status after program {:#010b}", status.inner); if !status.program_failed() { + if self.first_used_block_index.is_none() { + self.first_used_block_index = Some(b); + } if self.last_used_block_index.is_none() { self.last_used_block_index = Some(b); } else if let Some(last_used_block_index) = self.last_used_block_index { From 3a362a56602c54e7f62cf869ab731d77365d49c9 Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 09:00:09 +1200 Subject: [PATCH 07/12] remove some debugging --- src/core1_sub_tasks.rs | 2 +- src/onboard_flash.rs | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index 00c143c..eef0859 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -153,7 +153,7 @@ pub fn offload_flash_storage_and_events( let mut counter = timer.get_counter(); // TODO: Could speed this up slightly using cache_random_read interleaving on flash storage. // Probably doesn't matter though. - let previous_block = Some(10); + while has_file { let mut file_start = true; let mut part_count = 0; diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index cad40e8..295761a 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -626,15 +626,7 @@ impl OnboardFlash { let length = self.current_page.page_bytes_used(); let crc = self.current_page.page_crc(); let is_last_page_for_file = self.current_page.is_last_page_for_file(); - // if is_last_page_for_file { - // info!( - // "Got last file {}:{} file start is {} previous file start is {}", - // self.current_block_index, - // self.current_page_index, - // self.current_page.file_start(), - // self.current_page.previous_file_start() - // ); - // } + let block = self.current_block_index; let page = self.current_page_index; self.advance_file_cursor(is_last_page_for_file); From ceb83ee925915ae04783d2f15da51f556c6c4536 Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 10:03:17 +1200 Subject: [PATCH 08/12] remove prints --- src/core1_sub_tasks.rs | 6 ------ src/onboard_flash.rs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index eef0859..e201837 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -177,12 +177,6 @@ pub fn offload_flash_storage_and_events( crate::unreachable!("Invalid file transfer state"); }; - if file_start { - info!("Offload start is {}:{}", block_index, page_index) - } - if is_last { - info!("Got last file {}:{}", block_index, page_index); - } let crc_check = Crc::::new(&CRC_16_XMODEM); let current_crc = crc_check.checksum(&part); if current_crc != crc { diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index 295761a..73f3863 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -316,7 +316,7 @@ pub struct OnboardFlash { dma_channel_2: Option>, record_to_flash: bool, pub payload_buffer: Option<&'static mut [u8; 2115]>, - pub file_start: Option, + pub file_start: Option, //start of currently writing file, or last written } /// Each block is made up 64 pages of 2176 bytes. 139,264 bytes per block. /// Each page has a 2048 byte data storage section and a 128byte spare area for ECC codes. From 62f2fa4ed81544571bbfbab30d9a8fac1f8e7beb Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 15:28:39 +1200 Subject: [PATCH 09/12] rename variables --- src/core0_audio.rs | 2 +- src/core1_sub_tasks.rs | 12 ++++++++--- src/core1_task.rs | 2 +- src/event_logger.rs | 6 +++--- src/onboard_flash.rs | 48 +++++++++++++++++++++--------------------- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/core0_audio.rs b/src/core0_audio.rs index de46b6c..b40f866 100644 --- a/src/core0_audio.rs +++ b/src/core0_audio.rs @@ -672,7 +672,7 @@ fn should_offload_audio_recordings( info!("Offloading as logger is nearly full"); return true; } - if flash_storage.file_start.is_none() { + if flash_storage.file_start_block.is_none() { //one off info!("Offloading as previous file system version"); return true; diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index e201837..391a0e5 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -177,6 +177,12 @@ pub fn offload_flash_storage_and_events( crate::unreachable!("Invalid file transfer state"); }; + if file_start { + info!("Offload start is {}:{}", block_index, page_index) + } + if is_last { + info!("Got last file {}:{}", block_index, page_index); + } let crc_check = Crc::::new(&CRC_16_XMODEM); let current_crc = crc_check.checksum(&part); if current_crc != crc { @@ -246,7 +252,7 @@ pub fn offload_flash_storage_and_events( if !file_ended { info!( "Incomplete file at block {} erasing", - flash_storage.file_start + flash_storage.file_start_block ); if watchdog.is_some() { watchdog.as_mut().unwrap().feed(); @@ -254,7 +260,7 @@ pub fn offload_flash_storage_and_events( if let Err(e) = flash_storage.erase_last_file() { event_logger.log_event( LoggerEvent::new( - LoggerEventKind::EmptyErase, + LoggerEventKind::ErasePartialOrCorruptRecording, time.get_timestamp_micros(&timer), ), flash_storage, @@ -266,7 +272,7 @@ pub fn offload_flash_storage_and_events( if success { info!( "Completed file offload, transferred {} files start {} previous is {}", - file_count, flash_storage.file_start, flash_storage.previous_file_start + file_count, flash_storage.file_start_block, flash_storage.previous_file_start_block ); file_count != 0 } else { diff --git a/src/core1_task.rs b/src/core1_task.rs index df15ca3..fb05753 100644 --- a/src/core1_task.rs +++ b/src/core1_task.rs @@ -401,7 +401,7 @@ pub fn core_1_task( let should_offload = (has_files_to_offload && !device_config.time_is_in_recording_window(&synced_date_time.date_time_utc, &None)) || flash_storage.is_too_full_to_start_new_recordings() - || (has_files_to_offload && flash_storage.file_start.is_none()); + || (has_files_to_offload && flash_storage.file_start_block.is_none()); //means old file system offload once let should_offload = if !should_offload { let previous_offload_time = event_logger diff --git a/src/event_logger.rs b/src/event_logger.rs index c596db2..2391b6d 100644 --- a/src/event_logger.rs +++ b/src/event_logger.rs @@ -30,7 +30,7 @@ pub enum LoggerEventKind { AttinyCommError, Rp2040MissedAudioAlarm(u64), AudioRecordingFailed, - EmptyErase, + ErasePartialOrCorruptRecording, } impl Into for LoggerEventKind { @@ -58,7 +58,7 @@ impl Into for LoggerEventKind { AttinyCommError => 19, Rp2040MissedAudioAlarm(_) => 20, AudioRecordingFailed => 21, - EmptyErase => 22, + ErasePartialOrCorruptRecording => 22, } } } @@ -90,7 +90,7 @@ impl TryFrom for LoggerEventKind { 19 => Ok(AttinyCommError), 20 => Ok(Rp2040MissedAudioAlarm(0)), 21 => Ok(AudioRecordingFailed), - 22 => Ok(EmptyErase), + 22 => Ok(ErasePartialOrCorruptRecording), _ => Err(()), } } diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index 73f3863..99f2e5b 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -207,7 +207,7 @@ impl Page { self.user_metadata_1()[0] == 0 } - pub fn file_start(&self) -> Option { + pub fn file_start_block(&self) -> Option { let start = LittleEndian::read_u16(&self.user_metadata_1()[12..=13]); if start == u16::MAX { None @@ -216,7 +216,7 @@ impl Page { } } - pub fn previous_file_start(&self) -> Option<(u16)> { + pub fn previous_file_start_block(&self) -> Option<(u16)> { let block = LittleEndian::read_u16(&self.user_metadata_1()[14..=15]); if block == u16::MAX { None @@ -307,7 +307,7 @@ pub struct OnboardFlash { pub current_page_index: isize, pub current_block_index: isize, pub last_used_block_index: Option, - pub previous_file_start: Option, + pub previous_file_start_block: Option, pub first_used_block_index: Option, pub bad_blocks: [i16; 40], pub current_page: Page, @@ -316,7 +316,7 @@ pub struct OnboardFlash { dma_channel_2: Option>, record_to_flash: bool, pub payload_buffer: Option<&'static mut [u8; 2115]>, - pub file_start: Option, //start of currently writing file, or last written + pub file_start_block: Option, //start of currently writing file, or last written } /// Each block is made up 64 pages of 2176 bytes. 139,264 bytes per block. /// Each page has a 2048 byte data storage section and a 128byte spare area for ECC codes. @@ -354,8 +354,8 @@ impl OnboardFlash { dma_channel_2: Some(dma_channel_2), record_to_flash: should_record, payload_buffer: payload_buffer, - file_start: None, - previous_file_start: None, + file_start_block: None, + previous_file_start_block: None, } } pub fn init(&mut self) { @@ -422,7 +422,7 @@ impl OnboardFlash { if self.last_used_block_index.is_none() && self.first_used_block_index.is_some() { self.last_used_block_index = Some(block_index - 1); - self.file_start = self.prev_page.file_start(); + self.file_start_block = self.prev_page.file_start_block(); self.current_block_index = block_index; self.current_page_index = 0; @@ -536,8 +536,8 @@ impl OnboardFlash { } self.first_used_block_index = None; self.last_used_block_index = None; - self.file_start = None; - self.previous_file_start = None; + self.file_start_block = None; + self.previous_file_start_block = None; self.current_page_index = 0; self.current_block_index = 0; } @@ -561,18 +561,18 @@ impl OnboardFlash { //haven't used a block //havent written to file yet - if self.file_start.is_none() + if self.file_start_block.is_none() || self.last_used_block_index.is_none() - || self.last_used_block_index.unwrap() < self.file_start.unwrap() as isize + || self.last_used_block_index.unwrap() < self.file_start_block.unwrap() as isize { // self.last_used_block_index = self.previous_file_index; info!( "Nothing to erase start {} last used block {}", - self.file_start, self.last_used_block_index + self.file_start_block, self.last_used_block_index ); return Err("File hasn't been written too"); } - let start_block_index = self.file_start.unwrap() as isize; + let start_block_index = self.file_start_block.unwrap() as isize; info!( "Erasing last file {}:0 to {}", start_block_index, self.last_used_block_index @@ -586,8 +586,8 @@ impl OnboardFlash { return Err("Block erase failed"); } } - self.file_start = self.previous_file_start; - self.previous_file_start = None; + self.file_start_block = self.previous_file_start_block; + self.previous_file_start_block = None; if start_block_index == 0 { self.first_used_block_index = None; self.last_used_block_index = None; @@ -743,20 +743,20 @@ impl OnboardFlash { pub fn begin_offload_reverse(&mut self) -> bool { if let Some(last_block_index) = self.last_used_block_index { - if let Some(file_start) = self.file_start { + if let Some(file_start) = self.file_start_block { self.current_block_index = file_start as isize; self.current_page_index = 0; - self.previous_file_start = self.current_page.previous_file_start(); + self.previous_file_start_block = self.current_page.previous_file_start_block(); info!( "Set file start to {}:{} and previous {}", - self.current_block_index, 0, self.previous_file_start + self.current_block_index, 0, self.previous_file_start_block ); return true; } //old file system should only happen once self.current_block_index = self.find_start(last_block_index); self.current_page_index = 0; - self.file_start = Some(self.current_block_index as u16); + self.file_start_block = Some(self.current_block_index as u16); info!( "Searched for file start found {}:{}", self.current_block_index, self.last_used_block_index @@ -902,13 +902,13 @@ impl OnboardFlash { pub fn start_file(&mut self, start_page: isize) -> isize { // CPTV always start writing a new file at page 1, reserving page 0 for when we come back // and write the header once we've finished writing the file. - self.previous_file_start = self.file_start; + self.previous_file_start_block = self.file_start_block; self.current_page_index = start_page; warn!( "Starting file at file block {}, page {} previous is {}", - self.current_block_index, self.current_page_index, self.previous_file_start + self.current_block_index, self.current_page_index, self.previous_file_start_block ); - self.file_start = Some(self.current_block_index as u16); + self.file_start_block = Some(self.current_block_index as u16); self.current_block_index } @@ -1164,8 +1164,8 @@ impl OnboardFlash { } //write file start block let space = &mut bytes[4..][0x820..=0x83f][12..=13]; - LittleEndian::write_u16(space, self.file_start.unwrap()); - if let Some(previous_start) = self.previous_file_start { + LittleEndian::write_u16(space, self.file_start_block.unwrap()); + if let Some(previous_start) = self.previous_file_start_block { //write previous file start could just write on first page if it matters let space = &mut bytes[4..][0x820..=0x83f][14..=15]; LittleEndian::write_u16(space, previous_start as u16); From 0ec5b7eacdbc9bc19972f6982d7235bd399f0410 Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 15:32:23 +1200 Subject: [PATCH 10/12] remove unused max --- src/onboard_flash.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index 99f2e5b..5de6f9d 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -14,7 +14,6 @@ use crate::bsp::pac::SPI1; use crate::rp2040_flash::PAGE_SIZE; use byteorder::{ByteOrder, LittleEndian}; -use core::char::MAX; use core::mem; use cortex_m::singleton; use crc::{Crc, CRC_16_XMODEM}; From e8ec80b06a01ab4499aca23b4c52df26383088b5 Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 15:34:03 +1200 Subject: [PATCH 11/12] rename again --- src/core0_audio.rs | 2 +- src/core1_sub_tasks.rs | 6 ++++-- src/core1_task.rs | 2 +- src/onboard_flash.rs | 49 +++++++++++++++++++++--------------------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/core0_audio.rs b/src/core0_audio.rs index b40f866..c4573f8 100644 --- a/src/core0_audio.rs +++ b/src/core0_audio.rs @@ -672,7 +672,7 @@ fn should_offload_audio_recordings( info!("Offloading as logger is nearly full"); return true; } - if flash_storage.file_start_block.is_none() { + if flash_storage.file_start_block_index.is_none() { //one off info!("Offloading as previous file system version"); return true; diff --git a/src/core1_sub_tasks.rs b/src/core1_sub_tasks.rs index 391a0e5..9848a68 100644 --- a/src/core1_sub_tasks.rs +++ b/src/core1_sub_tasks.rs @@ -252,7 +252,7 @@ pub fn offload_flash_storage_and_events( if !file_ended { info!( "Incomplete file at block {} erasing", - flash_storage.file_start_block + flash_storage.file_start_block_index ); if watchdog.is_some() { watchdog.as_mut().unwrap().feed(); @@ -272,7 +272,9 @@ pub fn offload_flash_storage_and_events( if success { info!( "Completed file offload, transferred {} files start {} previous is {}", - file_count, flash_storage.file_start_block, flash_storage.previous_file_start_block + file_count, + flash_storage.file_start_block_index, + flash_storage.previous_file_start_block_index ); file_count != 0 } else { diff --git a/src/core1_task.rs b/src/core1_task.rs index fb05753..1e023d6 100644 --- a/src/core1_task.rs +++ b/src/core1_task.rs @@ -401,7 +401,7 @@ pub fn core_1_task( let should_offload = (has_files_to_offload && !device_config.time_is_in_recording_window(&synced_date_time.date_time_utc, &None)) || flash_storage.is_too_full_to_start_new_recordings() - || (has_files_to_offload && flash_storage.file_start_block.is_none()); + || (has_files_to_offload && flash_storage.file_start_block_index.is_none()); //means old file system offload once let should_offload = if !should_offload { let previous_offload_time = event_logger diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index 5de6f9d..02e8908 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -206,7 +206,7 @@ impl Page { self.user_metadata_1()[0] == 0 } - pub fn file_start_block(&self) -> Option { + pub fn file_start_block_index(&self) -> Option { let start = LittleEndian::read_u16(&self.user_metadata_1()[12..=13]); if start == u16::MAX { None @@ -215,7 +215,7 @@ impl Page { } } - pub fn previous_file_start_block(&self) -> Option<(u16)> { + pub fn previous_file_start_block_index(&self) -> Option<(u16)> { let block = LittleEndian::read_u16(&self.user_metadata_1()[14..=15]); if block == u16::MAX { None @@ -306,7 +306,7 @@ pub struct OnboardFlash { pub current_page_index: isize, pub current_block_index: isize, pub last_used_block_index: Option, - pub previous_file_start_block: Option, + pub previous_file_start_block_index: Option, pub first_used_block_index: Option, pub bad_blocks: [i16; 40], pub current_page: Page, @@ -315,7 +315,7 @@ pub struct OnboardFlash { dma_channel_2: Option>, record_to_flash: bool, pub payload_buffer: Option<&'static mut [u8; 2115]>, - pub file_start_block: Option, //start of currently writing file, or last written + pub file_start_block_index: Option, //start of currently writing file, or last written } /// Each block is made up 64 pages of 2176 bytes. 139,264 bytes per block. /// Each page has a 2048 byte data storage section and a 128byte spare area for ECC codes. @@ -353,8 +353,8 @@ impl OnboardFlash { dma_channel_2: Some(dma_channel_2), record_to_flash: should_record, payload_buffer: payload_buffer, - file_start_block: None, - previous_file_start_block: None, + file_start_block_index: None, + previous_file_start_block_index: None, } } pub fn init(&mut self) { @@ -421,7 +421,7 @@ impl OnboardFlash { if self.last_used_block_index.is_none() && self.first_used_block_index.is_some() { self.last_used_block_index = Some(block_index - 1); - self.file_start_block = self.prev_page.file_start_block(); + self.file_start_block_index = self.prev_page.file_start_block_index(); self.current_block_index = block_index; self.current_page_index = 0; @@ -535,8 +535,8 @@ impl OnboardFlash { } self.first_used_block_index = None; self.last_used_block_index = None; - self.file_start_block = None; - self.previous_file_start_block = None; + self.file_start_block_index = None; + self.previous_file_start_block_index = None; self.current_page_index = 0; self.current_block_index = 0; } @@ -560,18 +560,18 @@ impl OnboardFlash { //haven't used a block //havent written to file yet - if self.file_start_block.is_none() + if self.file_start_block_index.is_none() || self.last_used_block_index.is_none() - || self.last_used_block_index.unwrap() < self.file_start_block.unwrap() as isize + || self.last_used_block_index.unwrap() < self.file_start_block_index.unwrap() as isize { // self.last_used_block_index = self.previous_file_index; info!( "Nothing to erase start {} last used block {}", - self.file_start_block, self.last_used_block_index + self.file_start_block_index, self.last_used_block_index ); return Err("File hasn't been written too"); } - let start_block_index = self.file_start_block.unwrap() as isize; + let start_block_index = self.file_start_block_index.unwrap() as isize; info!( "Erasing last file {}:0 to {}", start_block_index, self.last_used_block_index @@ -585,8 +585,8 @@ impl OnboardFlash { return Err("Block erase failed"); } } - self.file_start_block = self.previous_file_start_block; - self.previous_file_start_block = None; + self.file_start_block_index = self.previous_file_start_block_index; + self.previous_file_start_block_index = None; if start_block_index == 0 { self.first_used_block_index = None; self.last_used_block_index = None; @@ -742,20 +742,21 @@ impl OnboardFlash { pub fn begin_offload_reverse(&mut self) -> bool { if let Some(last_block_index) = self.last_used_block_index { - if let Some(file_start) = self.file_start_block { + if let Some(file_start) = self.file_start_block_index { self.current_block_index = file_start as isize; self.current_page_index = 0; - self.previous_file_start_block = self.current_page.previous_file_start_block(); + self.previous_file_start_block_index = + self.current_page.previous_file_start_block_index(); info!( "Set file start to {}:{} and previous {}", - self.current_block_index, 0, self.previous_file_start_block + self.current_block_index, 0, self.previous_file_start_block_index ); return true; } //old file system should only happen once self.current_block_index = self.find_start(last_block_index); self.current_page_index = 0; - self.file_start_block = Some(self.current_block_index as u16); + self.file_start_block_index = Some(self.current_block_index as u16); info!( "Searched for file start found {}:{}", self.current_block_index, self.last_used_block_index @@ -901,13 +902,13 @@ impl OnboardFlash { pub fn start_file(&mut self, start_page: isize) -> isize { // CPTV always start writing a new file at page 1, reserving page 0 for when we come back // and write the header once we've finished writing the file. - self.previous_file_start_block = self.file_start_block; + self.previous_file_start_block_index = self.file_start_block_index; self.current_page_index = start_page; warn!( "Starting file at file block {}, page {} previous is {}", - self.current_block_index, self.current_page_index, self.previous_file_start_block + self.current_block_index, self.current_page_index, self.previous_file_start_block_index ); - self.file_start_block = Some(self.current_block_index as u16); + self.file_start_block_index = Some(self.current_block_index as u16); self.current_block_index } @@ -1163,8 +1164,8 @@ impl OnboardFlash { } //write file start block let space = &mut bytes[4..][0x820..=0x83f][12..=13]; - LittleEndian::write_u16(space, self.file_start_block.unwrap()); - if let Some(previous_start) = self.previous_file_start_block { + LittleEndian::write_u16(space, self.file_start_block_index.unwrap()); + if let Some(previous_start) = self.previous_file_start_block_index { //write previous file start could just write on first page if it matters let space = &mut bytes[4..][0x820..=0x83f][14..=15]; LittleEndian::write_u16(space, previous_start as u16); From 970a31131fa105e860e9b14a5c421fb338801ee0 Mon Sep 17 00:00:00 2001 From: gferraro Date: Mon, 19 Aug 2024 16:45:53 +1200 Subject: [PATCH 12/12] fix offload by reading page --- src/onboard_flash.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/onboard_flash.rs b/src/onboard_flash.rs index 02e8908..3b5a3cf 100644 --- a/src/onboard_flash.rs +++ b/src/onboard_flash.rs @@ -743,6 +743,10 @@ impl OnboardFlash { pub fn begin_offload_reverse(&mut self) -> bool { if let Some(last_block_index) = self.last_used_block_index { if let Some(file_start) = self.file_start_block_index { + //read 1 as if incomplete 0 won't be writen too + self.read_page(file_start as isize, 1).unwrap(); + self.read_page_metadata(file_start as isize); + self.wait_for_all_ready(); self.current_block_index = file_start as isize; self.current_page_index = 0; self.previous_file_start_block_index = @@ -1170,6 +1174,10 @@ impl OnboardFlash { let space = &mut bytes[4..][0x820..=0x83f][14..=15]; LittleEndian::write_u16(space, previous_start as u16); } + info!( + "Written start {} previous start {}", + self.file_start_block_index, self.previous_file_start_block_index + ); //info!("Wrote user meta {:?}", bytes[4..][0x820..=0x83f][0..10]); } // info!(