added features

added source to record output; added directory processing; updated tests;
forensicmatt · Jun 2, 2019 · fcd6e73 · fcd6e73
1 parent 3372e2b
commit fcd6e73
Show file tree

Hide file tree

Showing 6 changed files with 63 additions and 38 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "rusty_usn"
 description = "A fast and cross platform USN Parser written in Rust that outputs to JSONL"
-version = "1.0.0"
+version = "1.1.0"
 authors = ["Matthew Seyer"]
 edition = "2018"
 homepage = "https://github.com/forensicmatt/RustyUsn"

diff --git a/README.md b/README.md
@@ -1,13 +1,13 @@
-[![Build Status](https://dev.azure.com/matthewseyer/matthewseyer/_apis/build/status/forensicmatt.RustyUsn?branchName=master)](https://dev.azure.com/matthewseyer/matthewseyer/_build/latest?definitionId=1&branchName=master)
+[![Build Status](https://dev.azure.com/matthewseyer/dfir/_apis/build/status/forensicmatt.RustyUsn?branchName=master)](https://dev.azure.com/matthewseyer/dfir/_build/latest?definitionId=1&branchName=master)
 # RustyUsn
 A fast and cross platform USN Parser written in Rust. Output is [JSONL](http://jsonlines.org/).
 
 This does not currently implement records for usn record version 3 and 4.
 
 ```
-rusty_usn 1.0.0
+rusty_usn 1.1.0
 Matthew Seyer <https://github.com/forensicmatt/RustyUsn>
-USN Parser written in Rust.
+USN Parser written in Rust. Output is JSONL.
 
 USAGE:
     rusty_usn.exe [OPTIONS]
@@ -18,16 +18,17 @@ FLAGS:
 
 OPTIONS:
     -d, --debug <DEBUG>        Debug level to use. [possible values: Off, Error, Warn, Info, Debug, Trace]
-    -s, --source <PATH>        The source to parse.
+    -s, --source <PATH>        The source to parse. If the source is a directory, the directoy will be recursed looking
+                               for any files that end with '$J'.
     -t, --threads <threads>    Sets the number of worker threads, defaults to number of CPU cores. [default: 0]
 ```
 
 ## Output
 Records are written to stdout as jsonl.
 
 ```
-{"_offset":3272,"record_length":88,"major_version":2,"minor_version":0,"file_reference":{"entry":254303,"sequence":3},"parent_reference":{"entry":174492,"sequence":2},"usn":1231031496,"timestamp":"2018-07-30 19:52:08.147137","reason":"USN_REASON_CLOSE | USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":38,"file_name_length":24,"file_name_offset":60,"file_name":"DEFAULT.LOG2"}
-{"_offset":3184,"record_length":88,"major_version":2,"minor_version":0,"file_reference":{"entry":203911,"sequence":2},"parent_reference":{"entry":174492,"sequence":2},"usn":1231031408,"timestamp":"2018-07-30 19:52:08.147137","reason":"USN_REASON_CLOSE | USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":38,"file_name_length":22,"file_name_offset":60,"file_name":"SYSTEM.LOG1"}
+{"_source":"D:\\Images\\CTF_DEFCON_2018\\Image3-Desktop\\KAPE\\E\\$Extend\\$J","_offset":34464,"record_length":88,"major_version":2,"minor_version":0,"file_reference":{"entry":114704,"sequence":2},"parent_reference":{"entry":202493,"sequence":3},"usn":1231062688,"timestamp":"2018-07-30T20:15:57.100221Z","reason":"USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":32,"file_name_length":24,"file_name_offset":60,"file_name":"settings.dat"}
+{"_source":"D:\\Images\\CTF_DEFCON_2018\\Image3-Desktop\\KAPE\\E\\$Extend\\$J","_offset":34368,"record_length":96,"major_version":2,"minor_version":0,"file_reference":{"entry":114893,"sequence":2},"parent_reference":{"entry":202493,"sequence":3},"usn":1231062592,"timestamp":"2018-07-30T20:15:57.100221Z","reason":"USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":38,"file_name_length":34,"file_name_offset":60,"file_name":"settings.dat.LOG1"}
 ```
 
 # Carve USN from Unallocated
@@ -65,27 +66,12 @@ D:\Tools\RustyTools>rg -U -c "" D:\Testing\unallocated-usn.jsonl
 ## Build
 All you need is a ```cargo build --release``` for compiling with Rust. Currently using Rust 1.36.0 Nightly.
 
-## Change Log
-#### RustyUsn 1.0.0 (2019-05-27)
-- Rewrite and removal of features
+# Change Log
+## [1.1.0] - 2019-06-01
+### Added
+ - `_source` to output
+ - directory processing
 
-#### RustyUsn 0.5.0 (2017-06-22)
-- Added JMES Query functionality (http://jmespath.org/)
-- Added JSONL output (http://jsonlines.org/)
-
-#### RustyUsn 0.4.1 (2017-04-05)
-- Now using r-winstructs for file references and timestamps. This means greater controll of serialization. references are now strings on json serialization.
-
-#### RustyUsn 0.4.0 (2017-03-13)
-- Removed CSV output, Added JSON output
-
-#### RustyUsn 0.3.0 (2017-02-10)
-- Added human readable flags by default and option for integer flags (-f --flag)
-
-#### RustyUsn 0.2.1 (2017-02-09)
-- Using buffering with the seek_bufread library for better File IO operations.
-
-#### RustyUsn 0.2.0 (2017-02-08)
-- Parse from STDIN with -p option
-- Added Tests
-- Internal Restructure
+## [1.0.0] - 2019-05-27
+### Changed
+- Rewrite and removal of features
diff --git a/examples/test_iter_records.rs b/examples/test_iter_records.rs
@@ -41,6 +41,7 @@ fn main() {
     ];
 
     let iter = usn::IterRecords::new(
+        String::from("Test source"),
         raw_buffer.iter().cloned().collect(),
         0,
         raw_buffer.len()

diff --git a/src/bin/rusty_usn.rs b/src/bin/rusty_usn.rs
@@ -25,7 +25,7 @@ fn make_app<'a, 'b>() -> App<'a, 'b> {
         .short("s")
         .long("source")
         .value_name("PATH")
-        .help("The source to parse.")
+        .help("The source to parse. If the source is a directory, the directoy will be recursed looking for any files that end with '$J'.")
         .takes_value(true);
 
     let thread_count = Arg::with_name("threads")
@@ -46,7 +46,7 @@ fn make_app<'a, 'b>() -> App<'a, 'b> {
     App::new("rusty_usn")
         .version(VERSION)
         .author("Matthew Seyer <https://github.com/forensicmatt/RustyUsn>")
-        .about("USN Parser written in Rust.")
+        .about("USN Parser written in Rust. Output is JSONL.")
         .arg(source_arg)
         .arg(thread_count)
         .arg(verbose)
@@ -117,6 +117,31 @@ fn is_directory(source: &str)->bool{
 }
 
 
+fn process_directory(directory: &str, options: &ArgMatches) {
+    for dir_reader in fs::read_dir(directory) {
+        for entry_result in dir_reader {
+            match entry_result {
+                Ok(entry) => {
+                    let path = entry.path();
+                    if path.is_file() {
+                        let path_string = path.into_os_string().into_string().unwrap();
+                        if path_string.to_lowercase().ends_with("$j"){
+                            process_file(&path_string, &options);
+                        }
+                    } else if path.is_dir(){
+                        let path_string = path.into_os_string().into_string().unwrap();
+                        process_directory(&path_string, &options);
+                    }
+                },
+                Err(error) => {
+                    eprintln!("Error reading {} [{:?}]", directory, error);
+                }
+            }
+        }
+    }
+}
+
+
 fn process_file(file_location: &str, options: &ArgMatches) {
     info!("processing {}", file_location);
 
@@ -181,8 +206,7 @@ fn main() {
     };
 
     if is_directory(source_location) {
-        eprintln!("directory as a source is not currently implemented.");
-        exit(-1);
+        process_directory(source_location, &options);
     } else {
         process_file(source_location, &options);
     }

diff --git a/src/record.rs b/src/record.rs
@@ -31,16 +31,19 @@ impl UsnRecord {
 
 #[derive(Serialize, Debug)]
 pub struct UsnEntry {
+    #[serde(rename="_source")]
+    pub source: String,
     #[serde(rename="_offset")]
     pub offset: u64,
     #[serde(flatten)]
     pub record: UsnRecord,
 }
 impl UsnEntry {
-    pub fn new<R: Read>(offset: u64, version: u16, mut reader: R)-> Result<UsnEntry, UsnError>{
+    pub fn new<R: Read>(source: String, offset: u64, version: u16, mut reader: R)-> Result<UsnEntry, UsnError>{
         let record = UsnRecord::new(version, &mut reader)?;
 
         Ok(UsnEntry {
+            source: source,
             offset: offset,
             record: record,
         })

diff --git a/src/usn.rs b/src/usn.rs
@@ -62,6 +62,7 @@ impl UsnParserSettings {
 
 pub struct UsnParser<T: ReadSeek> {
     inner_handle: T,
+    source: String,
     handle_size: u64,
     settings: UsnParserSettings
 }
@@ -71,13 +72,14 @@ impl UsnParser<File> {
         let file_handle = File::open(filename)?;
 
         Self::from_read_seek(
+            filename.to_string(),
             file_handle
         )
     }
 }
 
 impl <T: ReadSeek> UsnParser <T> {
-    pub fn from_read_seek(mut inner_handle: T) -> Result<Self, io::Error> {
+    pub fn from_read_seek(source: String, mut inner_handle: T) -> Result<Self, io::Error> {
         // We need to get the end offset to determine the size
         let end_offset = inner_handle.seek(SeekFrom::End(0))?;
 
@@ -86,6 +88,7 @@ impl <T: ReadSeek> UsnParser <T> {
 
         Ok( Self {
             inner_handle: inner_handle,
+            source: source,
             handle_size: end_offset,
             settings: UsnParserSettings::default()
         })
@@ -204,6 +207,7 @@ impl <'c, T: ReadSeek> Iterator for IterFileChunks <'c, T> {
             // Return data chunk
             return Some(
                 DataChunk{
+                    source: self.parser.source.to_owned(),
                     offset: current_offset,
                     search_size: self.search_size,
                     data: buffer
@@ -263,6 +267,7 @@ impl<T: ReadSeek> Iterator for IntoIterFileChunks<T> {
             // Return data chunk
             return Some(
                 DataChunk{
+                    source: self.parser.source.to_owned(),
                     offset: current_offset,
                     search_size: self.search_size,
                     data: buffer
@@ -277,6 +282,7 @@ impl<T: ReadSeek> Iterator for IntoIterFileChunks<T> {
 
 #[derive(Debug)]
 pub struct DataChunk {
+    source: String,
     offset: u64,
     search_size: usize,
     data: Vec<u8>
@@ -295,6 +301,7 @@ impl DataChunk {
 
     pub fn get_record_iterator(self) -> IterRecords {
         IterRecords::new(
+            self.source,
             self.data, 
             self.offset,
             self.search_size
@@ -303,13 +310,14 @@ impl DataChunk {
 }
 
 pub struct IterRecords {
+    source: String,
     block: Vec<u8>,
     start_offset: u64,
     match_offsets: Vec<u64>,
 }
 
 impl IterRecords {
-    pub fn new(block: Vec<u8>, start_offset: u64, search_size: usize) -> IterRecords {
+    pub fn new(source: String, block: Vec<u8>, start_offset: u64, search_size: usize) -> IterRecords {
         lazy_static! {
             static ref RE_USN: bytes::Regex = bytes::Regex::new(
                 "(?-u)..\x00\x00\x02\x00\x00\x00"
@@ -329,6 +337,7 @@ impl IterRecords {
         }
 
         IterRecords {
+            source,
             block,
             start_offset,
             match_offsets
@@ -378,7 +387,9 @@ impl Iterator for IterRecords {
 
                     // Parse entry
                     let entry = match UsnEntry::new(
-                        entry_offset, 2, 
+                        self.source.clone(),
+                        entry_offset, 
+                        2,
                         &self.block[start_of_hit as usize ..]
                     ) {
                         Ok(entry) => entry,