merge upstream

.github/pyinstaller/pyinstaller.spec

@@ -1,10 +1,19 @@
 # -*- mode: python -*-
 # Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import os.path
-import subprocess
+import sys
 
 import wcwidth
+import capa.rules.cache
 
+from pathlib import Path
+
+# SPECPATH is a global variable which points to .spec file path
+capa_dir = Path(SPECPATH).parent.parent
+rules_dir = capa_dir / 'rules'
+cache_dir = capa_dir / 'cache'
+
+if not capa.rules.cache.generate_rule_cache(rules_dir, cache_dir):
+    sys.exit(-1)
 
 a = Analysis(
     # when invoking pyinstaller from the project root,
@@ -26,7 +35,7 @@ a = Analysis(
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), "wcwidth"),
+        (Path(wcwidth.__file__).parent, "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.

.github/workflows/build.yml

@@ -52,8 +52,6 @@ jobs:
         run: python -m pip install --upgrade pip setuptools
       - name: Install capa with build requirements
         run: pip install -e .[build]
-      - name: Cache the rule set
-        run: python ./scripts/cache-ruleset.py ./rules/ ./cache/
       - name: Build standalone executable
         run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?

CHANGELOG.md

@@ -12,7 +12,7 @@
 ### Breaking Changes
 
 
-### New Rules (12)
+### New Rules (16)
 
 - impact/wipe-disk/delete-drive-layout-via-ioctl [email protected]
 - host-interaction/driver/interact-with-driver-via-ioctl [email protected]
@@ -26,6 +26,10 @@
 - nursery/check-file-permission-on-linux [email protected]
 - nursery/check-if-process-is-running-under-android-emulator-on-android [email protected]
 - nursery/map-or-unmap-memory-on-linux [email protected]
+- persistence/act-as-share-provider-dll [email protected]
+- persistence/act-as-windbg-extension [email protected]
+- persistence/act-as-time-provider-dll [email protected]
+- host-interaction/gui/window/hide/hide-graphical-window-from-taskbar [email protected]
 -
 
 ### Bug Fixes
@@ -46,6 +50,7 @@
 - ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06
 - ci: update binja version to stable 4.0 #2016 @xusheng6
 - ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions  #2020 @psahithireddy
+- ci: include rule caching in PyInstaller build process #2097 @s-ff
 - add deptry support #1497 @s-ff
 
 ### Raw diffs
@@ -300,7 +305,7 @@ For those that use capa as a library, we've introduced some limited breaking cha
 - [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
 
 ## v5.1.0
-capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.
+capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and an output color format that's easier to read.
 
 Over 25 capa rules have been added and improved.
 
@@ -1499,7 +1504,7 @@ The IDA Pro integration is now distributed as a real plugin, instead of a script
   - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%`
   - generally doing thing the "right way"
 
-How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
+How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
 
 Please refer to the plugin [readme](https://github.com/mandiant/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin.
 

capa/features/common.py

@@ -128,7 +128,7 @@ def __eq__(self, other):
 
     def __lt__(self, other):
         # implementing sorting by serializing to JSON is a huge hack.
-        # its slow, inelegant, and probably doesn't work intuitively;
+        # it's slow, inelegant, and probably doesn't work intuitively;
         # however, we only use it for deterministic output, so it's good enough for now.
 
         # circular import
@@ -227,7 +227,7 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
             if self.value in feature.value:
                 matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
@@ -322,7 +322,7 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
             if self.re.search(feature.value):
                 matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 

capa/features/extractors/ghidra/extractor.py

@@ -34,7 +34,7 @@ def __init__(self):
                 # https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/Program.html
                 #
                 # the hashes are stored in the database, not computed on the fly,
-                # so its probably not trivial to add SHA1.
+                # so it's probably not trivial to add SHA1.
                 sha1="",
                 sha256=capa.ghidra.helpers.get_file_sha256(),
             )

capa/features/freeze/__init__.py

@@ -485,7 +485,7 @@ def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str:
                     address=Address.from_capa(addr),
                     feature=feature_from_capa(feature),
                 )  # type: ignore
-                # Mypy is unable to recognise `basic_block` as a argument due to alias
+                # Mypy is unable to recognise `basic_block` as an argument due to alias
                 for feature, addr in extractor.extract_thread_features(p, t)
             ]
 

capa/ida/plugin/__init__.py

@@ -125,7 +125,7 @@ def install_icon():
         return False
 
     # resource leak here. need to call `ida_kernwin.free_custom_icon`?
-    # however, since we're not cycling this icon a lot, its probably ok.
+    # however, since we're not cycling this icon a lot, it's probably ok.
     # expect to leak exactly one icon per application load.
     icon = ida_kernwin.load_custom_icon(data=ICON)
 

capa/render/result_document.py

@@ -418,7 +418,7 @@ def from_capa(
                             # doc[locations] contains all matches for the given namespace.
                             # for example, the feature might be `match: anti-analysis/packer`
                             # which matches against "generic unpacker" and "UPX".
-                            # in this case, doc[locations] contains locations for *both* of thse.
+                            # in this case, doc[locations] contains locations for *both* of those.
                             #
                             # rule_matches contains the matches for the specific rule.
                             # this is a subset of doc[locations].

capa/render/vverbose.py

@@ -48,7 +48,7 @@ def hanging_indent(s: str, indent: int) -> str:
 def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address], indent: int):
     import capa.render.verbose as v
 
-    # its possible to have an empty locations array here,
+    # it's possible to have an empty locations array here,
     # such as when we're in MODE_FAILURE and showing the logic
     # under a `not` statement (which will have no matched locations).
     locations = sorted(locations)

capa/rules/__init__.py

@@ -859,7 +859,7 @@ def rec(statement):
                 # we'll give precedence to namespaces, and then assume if that does work,
                 #  that it must be a rule name.
                 #
-                # we don't expect any collisions between namespaces and rule names, but its possible.
+                # we don't expect any collisions between namespaces and rule names, but it's possible.
                 # most likely would be collision between top level namespace (e.g. `host-interaction`) and rule name.
                 # but, namespaces tend to use `-` while rule names use ` `. so, unlikely, but possible.
                 if statement.value in namespaces:
@@ -1536,7 +1536,7 @@ def rec(rule_name: str, node: Union[Feature, Statement]):
         # don't consider it easy, and therefore,
         # don't index any of its features.
         #
-        # otherwise, its an easy rule, and index its features
+        # otherwise, it's an easy rule, and index its features
         for rules_with_feature in rules_by_feature.values():
             rules_with_feature.difference_update(rules_with_hard_features)
         easy_rules_by_feature = rules_by_feature

capa/rules/cache.py

@@ -159,3 +159,25 @@ def load_cached_ruleset(cache_dir: Path, rule_contents: List[bytes]) -> Optional
         return None
     else:
         return cache.ruleset
+
+
+def generate_rule_cache(rules_dir: Path, cache_dir: Path) -> bool:
+    if not rules_dir.is_dir():
+        logger.error("rules directory %s does not exist", rules_dir)
+        return False
+
+    try:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        rules = capa.rules.get_rules([rules_dir], cache_dir)
+    except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
+        logger.error("%s", str(e))
+        return False
+
+    content = capa.rules.cache.get_ruleset_content(rules)
+    id = capa.rules.cache.compute_cache_identifier(content)
+    path = capa.rules.cache.get_cache_path(cache_dir, id)
+
+    assert path.exists()
+    logger.info("rules cache saved to: %s", path)
+
+    return True

pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "dnfile==0.14.1",
     "dncil==1.0.2",
     "pydantic==2.7.1",
-    "rich==13.4.2",
+    "rich==13.7.1",
     "humanize==4.9.0",
     "protobuf==5.27.0",
 ]
@@ -79,7 +79,7 @@ dev = [
     "flake8-simplify==0.21.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.4.5",
+    "ruff==0.4.7",
     "black==24.4.2",
     "isort==5.13.2",
     "mypy==1.10.0",
@@ -96,7 +96,7 @@ dev = [
     "types-psutil==5.8.23",
     "types_requests==2.32.0.20240523",
     "types-protobuf==5.26.0.20240422",
-    "deptry==0.14"
+    "deptry==0.16.1"
 ]
 build = [
     "pyinstaller==6.7.0",

scripts/capa2yara.py

@@ -194,7 +194,7 @@ def do_statement(s_type, kid):
             yara_strings += "\t$" + var_name + ' = "' + string + '" ascii wide' + convert_description(kid) + "\n"
             yara_condition += "\t$" + var_name + " "
         elif s_type == "api" or s_type == "import":
-            # research needed to decide if its possible in YARA to make a difference between api & import?
+            # research needed to decide if it's possible in YARA to make a difference between api & import?
 
             # https://github.com/mandiant/capa-rules/blob/master/doc/format.md#api
             api = kid.value

tests/test_ghidra_features.py

@@ -6,7 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 """
-Must invoke this script from within the Ghidra Runtime Enviornment
+Must invoke this script from within the Ghidra Runtime Environment
 """
 import sys
 import logging