diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index da517c782..200c2b72e 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -27,7 +27,7 @@ jobs: uses: taiki-e/install-action@cargo-llvm-cov - name: Generate code coverage - run: cargo llvm-cov --workspace --lib --lcov --output-path lcov.info + run: cargo llvm-cov --features=magic-module --workspace --lib --lcov --output-path lcov.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d5a2596ca..faedd8e5f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,86 +12,91 @@ jobs: strategy: matrix: build: - - msrv - - stable - - nightly - - macos - - win-msvc - # - win-gnu - - no-default-features + - msrv + - stable + - nightly + - macos + - win-msvc + # - win-gnu + - no-default-features include: - - build: msrv - os: ubuntu-latest - rust: 1.73.0 - args: "" + - build: msrv + os: ubuntu-latest + rust: 1.73.0 + args: "--features=magic-module" - - build: stable - os: ubuntu-latest - rust: stable - args: "" + - build: stable + os: ubuntu-latest + rust: stable + args: "--features=magic-module" - - build: nightly - os: ubuntu-latest - rust: nightly - args: "" + - build: nightly + os: ubuntu-latest + rust: nightly + args: "--features=magic-module" - - build: macos - os: macos-latest - rust: stable - args: "" + - build: macos + os: macos-latest + rust: stable + args: "" - - build: win-msvc - os: windows-latest - rust: stable - args: "" + - build: win-msvc + os: windows-latest + rust: stable + args: "" + + # Tests for the `stable-x86_64-pc-windows-gnu` toolchain disabled + # due to https://github.com/VirusTotal/yara-x/issues/29 + # + # - build: win-gnu + # os: windows-latest + # rust: stable-x86_64-gnu + # args: "" - # Tests for the `stable-x86_64-pc-windows-gnu` toolchain disabled - # due to https://github.com/VirusTotal/yara-x/issues/29 - # - # - build: win-gnu - # os: windows-latest - # rust: stable-x86_64-gnu - # args: "" - - - build: no-default-features - os: ubuntu-latest - rust: stable - args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,string-module,time-module,hash-module,macho-module,math-module,lnk-module,elf-module,pe-module,dotnet-module,console-module" + - build: no-default-features + os: ubuntu-latest + rust: stable + args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,string-module,time-module,hash-module,macho-module,magic-module,math-module,lnk-module,elf-module,pe-module,dotnet-module,console-module" steps: - - name: Checkout sources - uses: actions/checkout@v4 + - name: Checkout sources + uses: actions/checkout@v4 - - name: Setup cache - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Setup cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Install OpenSSL - id: vcpkg - if: runner.os == 'Windows' - uses: johnwason/vcpkg-action@v5 - with: - pkgs: openssl - triplet: x64-windows-release - token: ${{ github.token }} + - name: Install dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y libssl-dev libmagic-dev - - name: Set OPENSSL_DIR environment variable - if: runner.os == 'Windows' - shell: bash - run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-release" >> $GITHUB_ENV + - name: Install dependencies + id: vcpkg + if: runner.os == 'Windows' + uses: johnwason/vcpkg-action@v5 + with: + pkgs: openssl + triplet: x64-windows-release + token: ${{ github.token }} - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.rust }} + - name: Set OPENSSL_DIR environment variable + if: runner.os == 'Windows' + shell: bash + run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-release" >> $GITHUB_ENV - - name: Run cargo test - run: cargo test --all-targets ${{ matrix.args }} - env: - RUSTFLAGS: -Awarnings # Allow all warnings + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - name: Run cargo test + run: cargo test --all-targets ${{ matrix.args }} + env: + RUSTFLAGS: -Awarnings # Allow all warnings \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index f2181b4c4..d81256192 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1954,6 +1954,28 @@ dependencies = [ "libc", ] +[[package]] +name = "magic" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a200ae03df8c3dce7a963f6eeaac8feb41bf9001cb7e5ab22e3205aec2f0373d" +dependencies = [ + "bitflags 2.4.2", + "libc", + "magic-sys", + "thiserror", +] + +[[package]] +name = "magic-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff86ae08895140d628119d407d568f3b657145ee8c265878064f717534bb3bc" +dependencies = [ + "libc", + "vcpkg", +] + [[package]] name = "maplit" version = "1.0.2" @@ -3500,6 +3522,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3648,17 +3676,6 @@ dependencies = [ "semver 1.0.21", ] -[[package]] -name = "wasmparser" -version = "0.201.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e5df6dba6c0d7fafc63a450f1738451ed7a0b52295d83e868218fa286bf708" -dependencies = [ - "bitflags 2.4.2", - "indexmap 2.2.2", - "semver 1.0.21", -] - [[package]] name = "wasmprinter" version = "0.2.78" @@ -3669,16 +3686,6 @@ dependencies = [ "wasmparser 0.121.0", ] -[[package]] -name = "wasmprinter" -version = "0.201.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a67e66da702706ba08729a78e3c0079085f6bfcb1a62e4799e97bbf728c2c265" -dependencies = [ - "anyhow", - "wasmparser 0.201.0", -] - [[package]] name = "wasmtime" version = "18.0.2" @@ -3835,7 +3842,7 @@ dependencies = [ "thiserror", "wasm-encoder 0.41.0", "wasmparser 0.121.0", - "wasmprinter 0.2.78", + "wasmprinter", "wasmtime-component-util", "wasmtime-types", ] @@ -4422,6 +4429,7 @@ dependencies = [ "lingua", "linkme", "log", + "magic", "md5", "memchr", "memx", @@ -4445,7 +4453,6 @@ dependencies = [ "tlsh-fixed", "uuid", "walrus", - "wasmprinter 0.201.0", "wasmtime", "yansi 0.5.1", "yara", diff --git a/Cargo.toml b/Cargo.toml index 1a5fe488a..bd270d27b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ bincode = "1.3.3" bitmask = "0.5.0" bitvec = "1.0.1" bstr = "1.8.0" +cbindgen = "0.26.0" chrono = "0.4.34" clap = "4.4.8" crc32fast = "1.3.2" @@ -62,6 +63,7 @@ sha2 = "0.10.7" # Using tlsh-fixed instead of tlsh because tlsh-fixed includes a fix for this # issue: https://github.com/1crcbl/tlsh-rs/issues/2. tlsh-fixed = "0.1.1" +magic = "0.16.2" memchr = "2.6.4" memx = "0.1.28" nom = "7.1.3" @@ -103,4 +105,4 @@ yara-x-proto-yaml = { path = "proto-yaml" } [profile.release-lto] inherits = "release" lto = true -codegen-units = 1 +codegen-units = 1 \ No newline at end of file diff --git a/capi/Cargo.toml b/capi/Cargo.toml index e5d2b6c72..60d2560d1 100644 --- a/capi/Cargo.toml +++ b/capi/Cargo.toml @@ -18,4 +18,4 @@ crate-type = ["staticlib", "cdylib"] yara-x = { workspace = true } [build-dependencies] -cbindgen = "0.26.0" \ No newline at end of file +cbindgen = { workspace = true } \ No newline at end of file diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8781ec9dc..7f9dc5f05 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -38,10 +38,10 @@ logging = ["dep:log", "dep:env_logger"] [dependencies] ascii_tree = { workspace = true } anyhow = { workspace = true } -clap = { workspace = true, features=["cargo", "derive"] } +clap = { workspace = true, features = ["cargo", "derive"] } globwalk = { workspace = true } enable-ansi-support = { workspace = true } -env_logger = { workspace = true , optional = true, features = ["auto-color"] } +env_logger = { workspace = true, optional = true, features = ["auto-color"] } log = { workspace = true, optional = true } protobuf = { workspace = true } protobuf-json-mapping = { workspace = true } @@ -55,7 +55,7 @@ yara-x-fmt = { workspace = true } colored_json = "4.0.0" crossbeam = "0.8.2" crossterm = "0.27.0" -pprof = { version = "0.13.0", features = ["flamegraph"], optional=true } +pprof = { version = "0.13.0", features = ["flamegraph"], optional = true } strum_macros = "0.25" superconsole = "0.2.0" wild = "2.1.0" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index bc345a381..fa0d3807d 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -96,6 +96,12 @@ macho-module = [ "dep:roxmltree", ] +# The `magic` allows recognizing file types based on the output of the +# Unix `file` command. This feature is disabled by default. +magic-module = [ + "dep:magic" +] + # The `math` module. math-module = [] @@ -166,6 +172,7 @@ log = { workspace = true, optional = true } md5 = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } +magic = { workspace = true, optional = true } memchr = { workspace = true } memx = { workspace = true } nom = { workspace = true, optional = true } @@ -205,11 +212,10 @@ globwalk = { workspace = true } goldenfile = { workspace = true } ihex = "3.0.0" pretty_assertions = { workspace = true } -wasmprinter = "0.201.0" yara = { version = "0.26.0", features = ["vendored"] } yara-x-proto-yaml = { workspace = true } zip = "0.6.6" [[bench]] name = "benches" -harness = false +harness = false \ No newline at end of file diff --git a/lib/src/modules/magic/mod.rs b/lib/src/modules/magic/mod.rs new file mode 100644 index 000000000..78a257336 --- /dev/null +++ b/lib/src/modules/magic/mod.rs @@ -0,0 +1,59 @@ +/*! YARA module that uses [libmagic][1] for recognizing file types. + +This allows creating YARA rules that use the file type provided by [libmagic][1]. + +[1]: https://man7.org/linux/man-pages/man3/libmagic.3.html + */ + +use crate::modules::prelude::*; +use crate::modules::protos::magic::*; + +#[cfg(test)] +mod tests; + +thread_local! { + static MAGIC: magic::Cookie = { + magic::Cookie::open(Default::default()) + .expect("initialized libmagic") + .load(&Default::default()) + .expect("loaded libmagic database") + }; +} + +#[module_main] +fn main(_data: &[u8]) -> Magic { + // Nothing to do, but we have to return our protobuf + Magic::new() +} + +#[module_export(name = "type")] +fn file_type(ctx: &mut ScanContext) -> Option { + Some(RuntimeString::from_slice( + ctx, + get_type(ctx.scanned_data()).as_bytes(), + )) +} + +#[module_export(name = "mime_type")] +fn mime_type(ctx: &mut ScanContext) -> Option { + Some(RuntimeString::from_slice( + ctx, + get_mime_type(ctx.scanned_data()).as_bytes(), + )) +} + +fn get_type(data: &[u8]) -> String { + MAGIC + .with(|magic| magic.set_flags(Default::default())) + .expect("set libmagic options"); + + MAGIC.with(|magic| magic.buffer(data)).expect("libmagic didn't break") +} + +fn get_mime_type(data: &[u8]) -> String { + MAGIC + .with(|magic| magic.set_flags(magic::cookie::Flags::MIME_TYPE)) + .expect("set libmagic options"); + + MAGIC.with(|magic| magic.buffer(data)).expect("libmagic didn't break") +} diff --git a/lib/src/modules/magic/tests/mod.rs b/lib/src/modules/magic/tests/mod.rs new file mode 100644 index 000000000..1464601fa --- /dev/null +++ b/lib/src/modules/magic/tests/mod.rs @@ -0,0 +1,33 @@ +use pretty_assertions::assert_eq; + +#[test] +fn get_filetype() { + let data = b"Maestro\r"; + let expected = "RISC OS music file"; + assert_eq!(expected, crate::modules::magic::get_type(data)) +} + +#[test] +fn get_mimetype() { + let expected = "text/plain"; + assert_eq!( + expected, + crate::modules::magic::get_mime_type(expected.as_bytes()) + ) +} + +#[test] +fn e2e_test() { + let rules = crate::compile( + r#" + import "magic" + rule t {condition: magic.type() == "RISC OS music file"} + "#, + ) + .unwrap(); + + let mut scanner = crate::scanner::Scanner::new(&rules); + let results = scanner.scan(b"Maestro\r").unwrap(); + + assert_eq!(results.matching_rules().len(), 1); +} diff --git a/lib/src/modules/modules.rs b/lib/src/modules/modules.rs index 28db9baf6..7ad936dbd 100644 --- a/lib/src/modules/modules.rs +++ b/lib/src/modules/modules.rs @@ -15,6 +15,8 @@ mod dotnet; mod lnk; #[cfg(feature = "hash-module")] mod hash; +#[cfg(feature = "magic-module")] +mod magic; #[cfg(feature = "math-module")] mod math; #[cfg(feature = "test_proto2-module")] diff --git a/lib/src/modules/protos/magic.proto b/lib/src/modules/protos/magic.proto new file mode 100644 index 000000000..50b2cb6df --- /dev/null +++ b/lib/src/modules/protos/magic.proto @@ -0,0 +1,16 @@ +syntax = "proto2"; + +import "yara.proto"; + +package magic; + +option (yara.module_options) = { + name : "magic" + root_message: "magic.Magic" + rust_module: "magic" + cargo_feature: "magic-module" +}; + +message Magic { + // This module contains only exported functions, and doesn't return any data +} \ No newline at end of file diff --git a/lib/src/wasm/builder.rs b/lib/src/wasm/builder.rs index 41a72df1e..13c1610af 100644 --- a/lib/src/wasm/builder.rs +++ b/lib/src/wasm/builder.rs @@ -428,82 +428,3 @@ impl WasmModuleBuilder { } } } - -#[cfg(test)] -mod tests { - use crate::wasm::builder::WasmModuleBuilder; - use itertools::Itertools; - use pretty_assertions::assert_eq; - - #[test] - fn module_builder() { - let mut builder = WasmModuleBuilder::new(); - - builder.namespaces_per_func(2); - builder.rules_per_func(2); - - builder.new_namespace(); - builder.new_global_rule(); - builder.new_rule().i32_const(4); - - builder.new_namespace(); - builder.new_rule().i32_const(5); - - builder.new_namespace(); - builder.new_rule().i32_const(6); - - let mut module = builder.build(); - let wasm = module.emit_wasm(); - let text = wasmprinter::print_bytes(wasm).unwrap(); - - // Remove all lines that start with " (type" or " (import". These - // lines are not relevant to this test, we are interested in - // the functions only. Also the order of imports is not stable - // across compiler versions. - let text = text - .split('\n') - .filter(|l| { - !l.starts_with(" (type") && !l.starts_with(" (import") - }) - .join("\n"); - - assert_eq!( - text, - r#"(module - (func (;164;) (type 1) (result i32) - i32.const 0 - global.set 2 - i32.const 0 - global.set 3 - call 165 - call 166 - global.get 3 - ) - (func (;165;) (type 0) - block ;; label = @1 - call 167 - end - block ;; label = @1 - call 168 - end - ) - (func (;166;) (type 0) - block ;; label = @1 - call 169 - end - ) - (func (;167;) (type 0) - i32.const 4 - ) - (func (;168;) (type 0) - i32.const 5 - ) - (func (;169;) (type 0) - i32.const 6 - ) - (export "main" (func 164)) -) -"# - ); - } -} diff --git a/py/Cargo.toml b/py/Cargo.toml index 5228cb845..4ae73e367 100644 --- a/py/Cargo.toml +++ b/py/Cargo.toml @@ -13,7 +13,7 @@ doc = false crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.2", features = ["abi3", "abi3-py38", "extension-module"]} +pyo3 = { version = "0.19.2", features = ["abi3", "abi3-py38", "extension-module"] } pyo3-file = "0.7.0" protobuf-json-mapping = { workspace = true }