Skip to content

Commit

Permalink
Merge pull request #5 from forensicmatt/rewrite
Browse files Browse the repository at this point in the history
Rewrite
  • Loading branch information
forensicmatt authored May 29, 2019
2 parents 4ca7d79 + cd55b24 commit fb59c1b
Show file tree
Hide file tree
Showing 20 changed files with 1,071 additions and 784 deletions.
55 changes: 26 additions & 29 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,35 +1,32 @@
[package]
name = "RustyUsn"
version = "0.5.0"
authors = ["Matthew Seyer <https://github.com/forensicmatt/RustyUsn>"]

[lib]
name = "rustyusn"
crate-type = ["cdylib","lib"]
name = "rusty_usn"
description = "A fast and cross platform USN Parser written in Rust that outputs to JSONL"
version = "1.0.0"
authors = ["Matthew Seyer - Github: https://github.com/forensicmatt - Twitter: @forensic_matt"]
edition = "2018"
homepage = "https://github.com/forensicmatt/RustyUsn"
repository = "https://github.com/forensicmatt/RustyUsn"
license = "Apache-2.0"
readme = "README.md"

[dependencies]
lazy_static = "*"
regex = "0.1.80"
byteorder = "*"
clap = "*"
time = "*"
seek_bufread = "~1.2"
bitflags = "0.7"
serde = "0.9"
serde_derive = "0.9"
serde_json = "0.9"
jmespath = "^0.1.1"

[dependencies.r-winstructs]
version = "*"
branch = "master"
git = "https://github.com/forensicmatt/r-winstructs"
log = "0.4"
fern = "0.5"
chrono = "0.4"
regex = "1"
lazy_static = "1.3.0"
bitflags = "1.0"
encoding = "0.2"
serde = "1.0"
serde_json = "1.0"
byteorder = "1.3.1"
winstructs = {git = "https://github.com/omerbenamram/winstructs.git", branch = "master"}
rayon = {version = "1.0.3", optional = true}

[dependencies.chrono]
version = "*"
features = ["serde", "rustc-serialize"]
[features]
default = ["multithreading"]
multithreading = ["rayon"]

[dependencies.cpython]
version = "*"
default-features = false
features = ["python27-sys"]
[[bin]]
name = "rusty_usn"
84 changes: 39 additions & 45 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,77 +1,71 @@
# RustyUsn
A fast and cross platform USN Parser written in Rust that gives you the ability to query the records via [JMESPath](http://jmespath.org/) queries. Output is [JSONL](http://jsonlines.org/).
A fast and cross platform USN Parser written in Rust. Output is [JSONL](http://jsonlines.org/).

```
RusyUsn 0.5.0
rusty_usn 1.0.0
Matthew Seyer <https://github.com/forensicmatt/RustyUsn>
USN Parser written in Rust.
USAGE:
RustyUsn.exe [FLAGS] [OPTIONS] --source <PATH>
rusty_usn.exe [OPTIONS]
FLAGS:
-b, --bool_expr JMES Query as bool only. (Prints whole record if true.)
-f, --flags Print flags as integers and not strings
-h, --help Prints help information
-r, --nonest Do not use nested references.
-p, --pipe Input from piped stdin
-V, --version Prints version information
-v, --verbose Verbose output for debug
-h, --help Prints help information
-V, --version Prints version information
OPTIONS:
-q, --query <QUERY> JMES Query
-s, --source <PATH> The USN journal file or folder with journals to parse.
-d, --debug <DEBUG> Debug level to use. [possible values: Off, Error, Warn, Info, Debug, Trace]
-s, --source <PATH> The source to parse.
-t, --threads <threads> Sets the number of worker threads, defaults to number of CPU cores. [default: 0]
```

## Output
The output is written to stdout as a json list of records.
Records are written to stdout as jsonl.

```
// DEFALUT OUTPUT
RustyUsn>target\release\RusyUsn.exe -s testdata\record.usn
{"record_length":96,"major_version":2,"minor_version":0,"file_reference_number":{"reference":"10477624533077459059","entry":115,"sequence":37224},"parent_file_reference_number":{"reference":"1970324837116475","entry":141883,"sequence":7},"usn":20342374400,"timestamp":"2013-10-19 12:16:53.276","reason":"USN_REASON_DATA_EXTEND","source_info":"","security_id":0,"file_attributes":8224,"file_name_length":32,"file_name_offset":60,"file_name":"BTDevManager.log"}
// DO NOT USE NESTED FILE REFERENCES
RustyUsn>target\release\RustyUsn.exe --nonest -s testdata\record.usn
{"record_length":96,"major_version":2,"minor_version":0,"file_reference_number":"10477624533077459059","parent_file_reference_number":"1970324837116475","usn":20342374400,"timestamp":"2013-10-19 12:16:53.276","reason":"USN_REASON_DATA_EXTEND","source_info":"","security_id":0,"file_attributes":8224,"file_name_length":32,"file_name_offset":60,"file_name":"BTDevManager.log"}
// DISPLAY FLAGS AS INTEGERS
RustyUsn>target\release\RustyUsn.exe -f -s testdata\record.usn
{"record_length":96,"major_version":2,"minor_version":0,"file_reference_number":{"reference":"10477624533077459059","entry":115,"sequence":37224},"parent_file_reference_number":{"reference":"1970324837116475","entry":141883,"sequence":7},"usn":20342374400,"timestamp":"2013-10-19 12:16:53.276","reason":2,"source_info":0,"security_id":0,"file_attributes":8224,"file_name_length":32,"file_name_offset":60,"file_name":"BTDevManager.log"}
{"_offset":3272,"record_length":88,"major_version":2,"minor_version":0,"file_reference":{"entry":254303,"sequence":3},"parent_reference":{"entry":174492,"sequence":2},"usn":1231031496,"timestamp":"2018-07-30 19:52:08.147137","reason":"USN_REASON_CLOSE | USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":38,"file_name_length":24,"file_name_offset":60,"file_name":"DEFAULT.LOG2"}
{"_offset":3184,"record_length":88,"major_version":2,"minor_version":0,"file_reference":{"entry":203911,"sequence":2},"parent_reference":{"entry":174492,"sequence":2},"usn":1231031408,"timestamp":"2018-07-30 19:52:08.147137","reason":"USN_REASON_CLOSE | USN_REASON_DATA_OVERWRITE","source_info":"(empty)","security_id":0,"file_attributes":38,"file_name_length":22,"file_name_offset":60,"file_name":"SYSTEM.LOG1"}
```

## Query Records
### Reformating using JMES Query
# Carve USN from Unallocated
To extract unallocated from an image, use the Sleuthkit's `blkls` with the `-A` option and redirect to a file. Pass that file into rusty_usn.exe.

1. Use TSK to extract out unallocated data.
```
// REFORMAT JSON OUTPUT USING A JMES QUERY
RustyUsn>target\release\RustyUsn.exe -s testdata\record.usn -q "[timestamp,file_name,reason]"
["2013-10-19 12:16:53.276","BTDevManager.log","USN_REASON_DATA_EXTEND"]
D:\Tools\sleuthkit-4.6.6-win32\bin>mmls D:\Images\CTF_DEFCON_2018\Image3-Desktop\Desktop-Disk0.e01
DOS Partition Table
Offset Sector: 0
Units are in 512-byte sectors
Slot Start End Length Description
000: Meta 0000000000 0000000000 0000000001 Primary Table (#0)
001: ------- 0000000000 0001126399 0001126400 Unallocated
002: 000:000 0001126400 0103904587 0102778188 NTFS / exFAT (0x07)
003: ------- 0103904588 0103905279 0000000692 Unallocated
004: 000:001 0103905280 0104855551 0000950272 Unknown Type (0x27)
005: ------- 0104855552 0104857599 0000002048 Unallocated
D:\Tools\sleuthkit-4.6.6-win32\bin>blkls -A -o 1126400 D:\Images\CTF_DEFCON_2018\Image3-Desktop\Desktop-Disk0.e01 > D:\Images\CTF_DEFCON_2018\Image3-Desktop\Desktop-Disk0.unallocated
```
### Filtering using JMES Query
Using the `-b` option will make the query use a bool value to filter results.

2. Parse the unallocated extracted file with rust_usn.exe.
```
// FILTER BY AN EXTENTION
RustyUsn>target\release\RustyUsn.exe -s testdata\record.usn -b -q "ends_with(file_name,'.log')"
{"record_length":96,"major_version":2,"minor_version":0,"file_reference_number":{"reference":"10477624533077459059","entry":115,"sequence":37224},"parent_file_reference_number":{"reference":"1970324837116475","entry":141883,"sequence":7},"usn":20342374400,"timestamp":"2013-10-19 12:16:53.276","reason":"USN_REASON_DATA_EXTEND","source_info":"","security_id":0,"file_attributes":8224,"file_name_length":32,"file_name_offset":60,"file_name":"BTDevManager.log"}
// FILTER WHERE RECORD HAS FILE_DELETE FLAG AND NAME ENDS WITH PF (DELETED PREFETCH FILES)
RustyUsn>target\release\RustyUsn.exe -b -q "contains(reason,'USN_REASON_FILE_DELETE')&&ends_with(file_name,'.pf')" -s $UsnJrnl.$J
{"record_length":112,"major_version":2,"minor_version":0,"file_reference_number":{"reference":"1125899906890782","entry":48158,"sequence":4},"parent_file_reference_number":{"reference":"562949953700461","entry":279149,"sequence":2},"usn":20371582824,"timestamp":"2013-10-21 19:46:03.599","reason":"USN_REASON_CLOSE | USN_REASON_FILE_DELETE","source_info":"","security_id":0,"file_attributes":8224,"file_name_length":48,"file_name_offset":60,"file_name":"REGSVR32.EXE-1098A44D.pf"}
{"record_length":112,"major_version":2,"minor_version":0,"file_reference_number":{"reference":"2814749767126627","entry":20067,"sequence":10},"parent_file_reference_number":{"reference":"562949953700461","entry":279149,"sequence":2},"usn":20371582976,"timestamp":"2013-10-21 19:46:03.599","reason":"USN_REASON_CLOSE | USN_REASON_FILE_DELETE","source_info":"","security_id":0,"file_attributes":8224,"file_name_length":46,"file_name_offset":60,"file_name":"DLLHOST.EXE-BCD52255.pf"}
...
D:\Tools\RustyTools>rusty_usn.exe -s D:\Images\CTF_DEFCON_2018\Image3-Desktop\Desktop-Disk0.unallocated > D:\Testing\unallocated-usn.jsonl
```

## Carving
The idea is to be able to parse records from stdin. You can grab unallocated with the Sleuthkit's blkls. Currently this has failed with RustyUsn.exe dying in some tests. I think more error checks are needed.
3. Count records recovered.
```
blkls.exe -o OFFSET IMAGEPATH | RustyUsn.exe -p > carved_records.txt
D:\Tools\RustyTools>rg -U -c "" D:\Testing\unallocated-usn.jsonl
1558102
```

## Build
All you need is a ```cargo build --release``` for compiling with Rust. Currently using Rust 1.15.0 Nightly.
All you need is a ```cargo build --release``` for compiling with Rust. Currently using Rust 1.36.0 Nightly.

## Change Log
#### RustyUsn 1.0.0 (2019-05-27)
- Rewrite and removal of features

#### RustyUsn 0.5.0 (2017-06-22)
- Added JMES Query functionality (http://jmespath.org/)
- Added JSONL output (http://jsonlines.org/)
Expand Down
88 changes: 88 additions & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
trigger:
branches:
include: ['*']
tags:
include: ['*']

strategy:
matrix:
windows-stable:
imageName: 'vs2017-win2016'
target: 'x86_64-pc-windows-msvc'
rustup_toolchain: stable
mac-stable:
imageName: 'macos-10.13'
target: 'x86_64-apple-darwin'
rustup_toolchain: stable
linux-stable:
imageName: 'ubuntu-16.04'
target: 'x86_64-unknown-linux-gnu'
rustup_toolchain: stable

pool:
vmImage: $(imageName)

steps:
- script: |
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUSTUP_TOOLCHAIN
echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin"
displayName: Install rust
condition: ne( variables['Agent.OS'], 'Windows_NT' )
- script: |
curl -sSf -o rustup-init.exe https://win.rustup.rs
rustup-init.exe -y --default-toolchain %RUSTUP_TOOLCHAIN%
echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin"
displayName: Windows install rust
condition: eq( variables['Agent.OS'], 'Windows_NT' )
- script: cargo build --release
displayName: Cargo build
- script: cargo test
displayName: Cargo test
- bash: |
MY_TAG="$(Build.SourceBranch)"
MY_TAG=${MY_TAG#refs/tags/}
echo $MY_TAG
echo "##vso[task.setvariable variable=build.my_tag]$MY_TAG"
displayName: "Create tag variable"
- bash: |
DATE="$(date +%Y-%m-%d)"
echo "##vso[task.setvariable variable=build.date]$DATE"
displayName: "Create date variable"
- bash: |
echo "##vso[task.setvariable variable=build.binary_name]rusty_usn.exe"
displayName: "Create date variable"
condition: eq( variables['Agent.OS'], 'Windows_NT' )
- bash: |
echo "##vso[task.setvariable variable=build.binary_name]rusty_usn"
displayName: "Create date variable"
condition: ne( variables['Agent.OS'], 'Windows_NT' )
- task: CopyFiles@2
displayName: Copy assets
inputs:
sourceFolder: '$(Build.SourcesDirectory)/target/release'
contents: |
$(build.binary_name)
targetFolder: '$(Build.BinariesDirectory)/$(build.binary_name)'
- task: ArchiveFiles@2
displayName: Gather assets
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)/$(build.binary_name)'
archiveType: 'tar'
tarCompression: 'gz'
archiveFile: '$(Build.ArtifactStagingDirectory)/rusty_usn-$(build.my_tag)-$(TARGET).tar.gz'
- task: GithubRelease@0
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
inputs:
gitHubConnection: 'Github'
repositoryName: 'forensicmatt/RustyUsn'
action: 'edit'
target: '$(build.sourceVersion)'
tagSource: 'manual'
tag: '$(build.my_tag)'
assets: '$(Build.ArtifactStagingDirectory)/rusty_usn-$(build.my_tag)-$(TARGET).tar.gz'
title: '$(build.my_tag) - $(build.date)'
assetUploadMode: 'replace'
addChangeLog: false
3 changes: 0 additions & 3 deletions build.bat

This file was deleted.

52 changes: 0 additions & 52 deletions etc/mapping.usn.json

This file was deleted.

9 changes: 3 additions & 6 deletions examples/regex_example.rs → examples/test_iter.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#[macro_use] extern crate lazy_static;
extern crate regex;
use regex::bytes;

Expand Down Expand Up @@ -42,13 +41,11 @@ fn main() {
0x00,0x00,0x00,0x00,0x20,0x20,0x00,0x00
];

lazy_static! {
static ref RE: bytes::Regex = bytes::Regex::new(
let re = bytes::Regex::new(
"..\x00\x00(\x02)\x00\x00\x00"
).unwrap();
}

for hit in RE.find_iter(&raw_buffer[..]) {
println!("Hit at: {}",hit.0);
for hit in re.find_iter(&raw_buffer[..]) {
println!("Hit at: {}", hit.start());
}
}
Loading

0 comments on commit fb59c1b

Please sign in to comment.