Revert "Merge branch 'mandiant:master' into feature-static-api-names"

This reverts commit 84215ea, reversing
changes made to 97c8fd0.

.github/workflows/tests.yml

@@ -33,8 +33,6 @@ jobs:
         python-version: "3.8"
     - name: Install dependencies
       run: pip install -e .[dev]
-    - name: Lint with ruff
-      run: ruff check --config .github/ruff.toml .
     - name: Lint with isort
       run: isort --profile black --length-sort --line-width 120 --skip-glob "*_pb2.py" -c .
     - name: Lint with black
@@ -96,12 +94,9 @@ jobs:
       run: pytest -v tests/
 
   binja-tests:
-    name: Binary Ninja tests for ${{ matrix.python-version }}
+    name: Binary Ninja tests for ${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ubuntu-20.04
     needs: [code_style, rule_linter]
-    # do not run on forks, because:
-    # with the exception of GITHUB_TOKEN, secrets are not passed to the runner when a workflow is triggered from a forked repository.
-    if: "! github.event.pull_request.head.repo.fork"
     strategy:
       fail-fast: false
       matrix:

CHANGELOG.md

@@ -7,10 +7,8 @@
 
 ### Breaking Changes
 
-### New Rules (2)
+### New Rules (0)
 
-- load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
-- nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
 -
 
 ### Bug Fixes

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-795-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-794-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/features/extractors/binja/insn.py

@@ -439,7 +439,7 @@ def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index:
             return True
 
         value = right.value.value
-        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
+        if not (reg, value) in (("fsbase", 0x30), ("gsbase", 0x60)):
             return True
 
         results.append((Characteristic("peb access"), ih.address))

capa/features/extractors/ida/insn.py

@@ -73,7 +73,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
     """
     insn: idaapi.insn_t = ih.inner
 
-    if insn.get_canon_mnem() not in ("call", "jmp"):
+    if not insn.get_canon_mnem() in ("call", "jmp"):
         return
 
     # check calls to imported functions

capa/ida/plugin/form.py

@@ -536,7 +536,7 @@ def ida_hook_screen_ea_changed(self, widget, new_ea, old_ea):
         @param new_ea: destination ea
         @param old_ea: source ea
         """
-        if self.view_tabs.currentIndex() not in (0, 1):
+        if not self.view_tabs.currentIndex() in (0, 1):
             return
 
         if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
@@ -607,8 +607,7 @@ def ensure_capa_settings_rule_path(self):
         except UserCancelledError as e:
             capa.ida.helpers.inform_user_ida_ui("Analysis requires capa rules")
             logger.warning(
-                "You must specify a directory containing capa rules before running analysis.%s",
-                f"Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended).",
+                f"You must specify a directory containing capa rules before running analysis. Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended)."
             )
             return False
         except Exception as e:
@@ -706,8 +705,7 @@ def load_capa_results(self, new_analysis, from_cache):
 
                         capa.ida.helpers.inform_user_ida_ui("Cached results were generated using different capas rules")
                         logger.warning(
-                            "capa is showing you cached results from a previous analysis run.%s ",
-                            "Your rules have changed since and you should reanalyze the program to see new results.",
+                            "capa is showing you cached results from a previous analysis run. Your rules have changed since and you should reanalyze the program to see new results."
                         )
                         view_status_rules = "no rules matched for cache"
 

capa/main.py

@@ -255,8 +255,7 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
     if disable_progress:
         # do not use tqdm to avoid unnecessary side effects when caller intends
         # to disable progress completely
-        def pbar(s, *args, **kwargs):
-            return s
+        pbar = lambda s, *args, **kwargs: s
 
     functions = list(extractor.get_functions())
     n_funcs = len(functions)

scripts/capa2yara.py

@@ -59,8 +59,7 @@
 # -- https://github.com/mandiant/capa-rules/blob/master/collection/file-managers/gather-direct-ftp-information.yml
 # -- https://github.com/mandiant/capa-rules/blob/master/collection/browser/gather-firefox-profile-information.yml
 # - count(string    (1 rule: /executable/subfile/pe/contain-an-embedded-pe-file.yml)
-# - count(match( could be done by creating the referenced rule a 2nd time with the condition, that it hits x times
-# (only 1 rule: ./anti-analysis/anti-disasm/contain-anti-disasm-techniques.yml)
+# - count(match( could be done by creating the referenced rule a 2nd time with the condition, that it hits x times (only 1 rule: ./anti-analysis/anti-disasm/contain-anti-disasm-techniques.yml)
 # - it would be technically possible to get the "basic blocks" working, but the rules contain mostly other non supported statements in there => not worth the effort.
 
 # collect all converted rules to be able to check if we have needed sub rules for match:
@@ -129,8 +128,7 @@ def convert_capa_number_to_yara_bytes(number):
 
 
 def convert_rule_name(rule_name):
-    # yara rule names: "Identifiers must follow the same lexical conventions of the C programming language, they can contain any alphanumeric character and the underscore character
-    # but the first character cannot be a digit. Rule identifiers are case sensitive and cannot exceed 128 characters." so we replace any non-alphanum with _
+    # yara rule names: "Identifiers must follow the same lexical conventions of the C programming language, they can contain any alphanumeric character and the underscore character, but the first character cannot be a digit. Rule identifiers are case sensitive and cannot exceed 128 characters." so we replace any non-alphanum with _
     rule_name = re.sub(r"\W", "_", rule_name)
     rule_name = "capa_" + rule_name
 
@@ -199,8 +197,7 @@ def do_statement(s_type, kid):
 
                 # even looking for empty string in dll_regex doesn't work for some files (list below) with pe.imports so do just a string search
                 # yara_condition += '\tpe.imports(/.{0,30}/i, /' + api + '/) '
-                # 5fbbfeed28b258c42e0cfeb16718b31c, 2D3EDC218A90F03089CC01715A9F047F, 7EFF498DE13CC734262F87E6B3EF38AB,
-                # C91887D861D9BD4A5872249B641BC9F9, a70052c45e907820187c7e6bcdc7ecca, 0596C4EA5AA8DEF47F22C85D75AACA95
+                # 5fbbfeed28b258c42e0cfeb16718b31c, 2D3EDC218A90F03089CC01715A9F047F, 7EFF498DE13CC734262F87E6B3EF38AB, C91887D861D9BD4A5872249B641BC9F9, a70052c45e907820187c7e6bcdc7ecca, 0596C4EA5AA8DEF47F22C85D75AACA95
                 var_name = "api_" + var_names.pop(0)
 
                 # limit regex with word boundary \b but also search for appended A and W
@@ -289,17 +286,15 @@ def do_statement(s_type, kid):
 
             # all .* in the regexes of capa look like they should be maximum 100 chars so take 1000 to speed up rules and prevent yara warnings on poor performance
             regex = regex.replace(".*", ".{,1000}")
-            # strange: capa accepts regexes with unescaped /
-            # like - string: /com/exe4j/runtime/exe4jcontroller/i in capa-rules/compiler/exe4j/compiled-with-exe4j.yml, needs a fix for yara:
+            # strange: capa accepts regexes with unescaped / like - string: /com/exe4j/runtime/exe4jcontroller/i in capa-rules/compiler/exe4j/compiled-with-exe4j.yml, needs a fix for yara:
             # would assume that get_value_str() gives the raw string
             regex = re.sub(r"(?<!\\)/", r"\/", regex)
 
             # capa uses python regex which accepts /reg(|.exe)/ but yaras regex engine doesn't not => fix it
             # /reg(|.exe)/ => /reg(.exe)?/
             regex = re.sub(r"\(\|([^\)]+)\)", r"(\1)?", regex)
 
-            # change beginning of line to null byte, e.g. /^open => /\x00open
-            # (not word boundary because we're not looking for the beginning of a word in a text but usually a function name if there's ^ in a capa rule)
+            # change beginning of line to null byte, e.g. /^open => /\x00open (not word boundary because we're not looking for the beginning of a word in a text but usually a function name if there's ^ in a capa rule)
             regex = re.sub(r"^\^", r"\\x00", regex)
 
             # regex = re.sub(r"^\^", r"\\b", regex)
@@ -425,8 +420,7 @@ def do_statement(s_type, kid):
                         )
                         # remove last 'or'
                         # yara_condition = re.sub(r'\sor $', ' ', yara_condition)
-                        rule_comment += "This rule is incomplete because a branch inside an Or-statement had an unsupported feature and was skipped "
-                        rule_comment += "=> coverage is reduced compared to the original capa rule. "
+                        rule_comment += "This rule is incomplete because a branch inside an Or-statement had an unsupported feature and was skipped => coverage is reduced compared to the original capa rule. "
                         x += 1
                         incomplete = 1
                         continue
@@ -452,8 +446,7 @@ def do_statement(s_type, kid):
                         + str(depth)
                     )
 
-                    rule_comment += "This rule is incomplete because a branch inside an Or-statement had an unsupported feature and was skipped"
-                    rule_comment += "=> coverage is reduced compared to the original capa rule. "
+                    rule_comment += "This rule is incomplete because a branch inside an Or-statement had an unsupported feature and was skipped => coverage is reduced compared to the original capa rule. "
                     x += 1
                     incomplete = 1
                     continue
@@ -676,8 +669,7 @@ def convert_rules(rules, namespaces, cround, make_priv):
 
                 yara += "  condition:" + condition_header + yara_condition + "\n}"
 
-                # TODO: now the rule is finished and could be automatically checked with the capa-testfile(s) named in meta
-                # (doing it for all of them using yara-ci upload at the moment)
+                # TODO: now the rule is finished and could be automatically checked with the capa-testfile(s) named in meta (doing it for all of them using yara-ci upload at the moment)
                 output_yar(yara)
                 converted_rules.append(rule_name)
                 count_incomplete += incomplete
@@ -727,7 +719,7 @@ def main(argv=None):
         "// Rules from Mandiant's https://github.com/mandiant/capa-rules converted to YARA using https://github.com/mandiant/capa/blob/master/scripts/capa2yara.py by Arnim Rupp"
     )
     output_yar(
-        "// Beware: These are less rules than capa (because not all fit into YARA, stats at EOF) and is less precise e.g. capas function scopes are applied to the whole file"
+        "// Beware: These are less rules than capa (because not all fit into YARA, stats at EOF) and is less precise because e.g. capas function scopes are applied to the whole file"
     )
     output_yar(
         '// Beware: Some rules are incomplete because an optional branch was not supported by YARA. These rules are marked in a comment in meta: (search for "incomplete")'

setup.py

@@ -74,7 +74,6 @@
             "pytest-instafail==0.5.0",
             "pytest-cov==4.0.0",
             "pycodestyle==2.10.0",
-            "ruff==0.0.260",
             "black==23.3.0",
             "isort==5.11.4",
             "mypy==1.2.0",