feat: implement the magic module. (#91)

Mostly a copy of #12. After too many changes since the PR, it's easier to commit the code again than resolving the merge conflicts.
VirusTotal · Mar 13, 2024 · 5ec7aef · 5ec7aef
1 parent 7aee275
commit 5ec7aef
Show file tree

Hide file tree

Showing 13 changed files with 231 additions and 180 deletions.
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -27,7 +27,7 @@ jobs:
       uses: taiki-e/install-action@cargo-llvm-cov
 
     - name: Generate code coverage
-      run: cargo llvm-cov --workspace --lib --lcov --output-path lcov.info
+      run: cargo llvm-cov --features=magic-module --workspace --lib --lcov --output-path lcov.info
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -12,86 +12,91 @@ jobs:
     strategy:
       matrix:
         build:
-          - msrv
-          - stable
-          - nightly
-          - macos
-          - win-msvc
-          # - win-gnu
-          - no-default-features
+        - msrv
+        - stable
+        - nightly
+        - macos
+        - win-msvc
+        # - win-gnu
+        - no-default-features
         include:
-          - build: msrv
-            os: ubuntu-latest
-            rust: 1.73.0
-            args: ""
+        - build: msrv
+          os: ubuntu-latest
+          rust: 1.73.0
+          args: "--features=magic-module"
 
-          - build: stable
-            os: ubuntu-latest
-            rust: stable
-            args: ""
+        - build: stable
+          os: ubuntu-latest
+          rust: stable
+          args: "--features=magic-module"
 
-          - build: nightly
-            os: ubuntu-latest
-            rust: nightly
-            args: ""
+        - build: nightly
+          os: ubuntu-latest
+          rust: nightly
+          args: "--features=magic-module"
 
-          - build: macos
-            os: macos-latest
-            rust: stable
-            args: ""
+        - build: macos
+          os: macos-latest
+          rust: stable
+          args: ""
 
-          - build: win-msvc
-            os: windows-latest
-            rust: stable
-            args: ""
+        - build: win-msvc
+          os: windows-latest
+          rust: stable
+          args: ""
+
+        # Tests for the `stable-x86_64-pc-windows-gnu` toolchain disabled
+        # due to https://github.com/VirusTotal/yara-x/issues/29
+        #
+        # - build: win-gnu
+        #   os: windows-latest
+        #   rust: stable-x86_64-gnu
+        #   args: ""
 
-          # Tests for the `stable-x86_64-pc-windows-gnu` toolchain disabled
-          # due to https://github.com/VirusTotal/yara-x/issues/29
-          #
-          # - build: win-gnu
-          #   os: windows-latest
-          #   rust: stable-x86_64-gnu
-          #   args: ""
-
-          - build: no-default-features
-            os: ubuntu-latest
-            rust: stable
-            args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,string-module,time-module,hash-module,macho-module,math-module,lnk-module,elf-module,pe-module,dotnet-module,console-module"
+        - build: no-default-features
+          os: ubuntu-latest
+          rust: stable
+          args: "--package yara-x --no-default-features --features=test_proto2-module,test_proto3-module,string-module,time-module,hash-module,macho-module,magic-module,math-module,lnk-module,elf-module,pe-module,dotnet-module,console-module"
 
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v4
+    - name: Checkout sources
+      uses: actions/checkout@v4
 
-      - name: Setup cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - name: Setup cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Install OpenSSL
-        id: vcpkg
-        if: runner.os == 'Windows'
-        uses: johnwason/vcpkg-action@v5
-        with:
-          pkgs: openssl
-          triplet: x64-windows-release
-          token: ${{ github.token }}
+    - name: Install dependencies
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libssl-dev libmagic-dev
 
-      - name: Set OPENSSL_DIR environment variable
-        if: runner.os == 'Windows'
-        shell: bash
-        run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-release" >> $GITHUB_ENV
+    - name: Install dependencies
+      id: vcpkg
+      if: runner.os == 'Windows'
+      uses: johnwason/vcpkg-action@v5
+      with:
+        pkgs: openssl
+        triplet: x64-windows-release
+        token: ${{ github.token }}
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ matrix.rust }}
+    - name: Set OPENSSL_DIR environment variable
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "OPENSSL_DIR=${{ github.workspace }}\\vcpkg\\installed\\x64-windows-release" >> $GITHUB_ENV
 
-      - name: Run cargo test
-        run: cargo test --all-targets ${{ matrix.args }}
-        env:
-          RUSTFLAGS: -Awarnings # Allow all warnings
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ matrix.rust }}
 
+    - name: Run cargo test
+      run: cargo test --all-targets ${{ matrix.args }}
+      env:
+        RUSTFLAGS: -Awarnings # Allow all warnings
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -39,6 +39,7 @@ bincode = "1.3.3"
 bitmask = "0.5.0"
 bitvec = "1.0.1"
 bstr = "1.8.0"
+cbindgen = "0.26.0"
 chrono = "0.4.34"
 clap = "4.4.8"
 crc32fast = "1.3.2"
@@ -62,6 +63,7 @@ sha2 = "0.10.7"
 # Using tlsh-fixed instead of tlsh because tlsh-fixed includes a fix for this
 # issue: https://github.com/1crcbl/tlsh-rs/issues/2.
 tlsh-fixed = "0.1.1"
+magic = "0.16.2"
 memchr = "2.6.4"
 memx = "0.1.28"
 nom = "7.1.3"
@@ -103,4 +105,4 @@ yara-x-proto-yaml = { path = "proto-yaml" }
 [profile.release-lto]
 inherits = "release"
 lto = true
-codegen-units = 1
+codegen-units = 1
diff --git a/capi/Cargo.toml b/capi/Cargo.toml
@@ -18,4 +18,4 @@ crate-type = ["staticlib", "cdylib"]
 yara-x = { workspace = true }
 
 [build-dependencies]
-cbindgen = "0.26.0"
+cbindgen = { workspace = true }
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -38,10 +38,10 @@ logging = ["dep:log", "dep:env_logger"]
 [dependencies]
 ascii_tree = { workspace = true }
 anyhow = { workspace = true }
-clap = { workspace = true, features=["cargo", "derive"] }
+clap = { workspace = true, features = ["cargo", "derive"] }
 globwalk = { workspace = true }
 enable-ansi-support = { workspace = true }
-env_logger = { workspace = true , optional = true, features = ["auto-color"] }
+env_logger = { workspace = true, optional = true, features = ["auto-color"] }
 log = { workspace = true, optional = true }
 protobuf = { workspace = true }
 protobuf-json-mapping = { workspace = true }
@@ -55,7 +55,7 @@ yara-x-fmt = { workspace = true }
 colored_json = "4.0.0"
 crossbeam = "0.8.2"
 crossterm = "0.27.0"
-pprof = { version = "0.13.0", features = ["flamegraph"], optional=true }
+pprof = { version = "0.13.0", features = ["flamegraph"], optional = true }
 strum_macros = "0.25"
 superconsole = "0.2.0"
 wild = "2.1.0"
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
@@ -96,6 +96,12 @@ macho-module = [
     "dep:roxmltree",
 ]
 
+# The `magic` allows recognizing file types based on the output of the
+# Unix `file` command. This feature is disabled by default.
+magic-module = [
+    "dep:magic"
+]
+
 # The `math` module.
 math-module = []
 
@@ -166,6 +172,7 @@ log = { workspace = true, optional = true }
 md5 = { workspace = true, optional = true }
 sha1 = { workspace = true, optional = true }
 sha2 = { workspace = true, optional = true }
+magic = { workspace = true, optional = true }
 memchr = { workspace = true }
 memx = { workspace = true }
 nom = { workspace = true, optional = true }
@@ -205,11 +212,10 @@ globwalk = { workspace = true }
 goldenfile = { workspace = true }
 ihex = "3.0.0"
 pretty_assertions = { workspace = true }
-wasmprinter = "0.201.0"
 yara = { version = "0.26.0", features = ["vendored"] }
 yara-x-proto-yaml = { workspace = true }
 zip = "0.6.6"
 
 [[bench]]
 name = "benches"
-harness = false
+harness = false