Merge branch 'master' into master

CHANGELOG.md

@@ -9,13 +9,15 @@
 
 ### Breaking Changes
 
-### New Rules (6)
+### New Rules (8)
 
 - executable/pe/export/forwarded-export [email protected]
 - host-interaction/bootloader/get-uefi-variable [email protected]
 - host-interaction/bootloader/set-uefi-variable [email protected]
 - nursery/enumerate-device-drivers-on-linux @mr-tz
 - anti-analysis/anti-vm/vm-detection/check-for-foreground-window-switch [email protected]
+- linking/static/sqlite3/linked-against-cppsqlite3 [email protected]
+- linking/static/sqlite3/linked-against-sqlite3 [email protected]
 -
 
 ### Bug Fixes
@@ -24,6 +26,8 @@
 - linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin 
 - OS: detect Android ELF files #1705 @williballenthin
 - ELF: fix parsing of symtab #1704 @williballenthin
+- result document: don't use deprecated pydantic functions #1718 @williballenthin
+- pytest: don't mark IDA tests as pytest tests #1719 @williballenthin
 
 ### capa explorer IDA Pro plugin
 - fix unhandled exception when resolving rule path #1693 @mike-hunhoff

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-829-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-831-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/features/freeze/__init__.py

@@ -320,7 +320,7 @@ def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
     """deserialize a set of features (as a NullFeatureExtractor) from a string."""
     import capa.features.extractors.null as null
 
-    freeze = Freeze.parse_raw(s)
+    freeze = Freeze.model_validate_json(s)
     if freeze.version != 2:
         raise ValueError(f"unsupported freeze format version: {freeze.version}")
 

capa/ida/helpers.py

@@ -5,7 +5,6 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import json
 import logging
 import datetime
 import contextlib
@@ -223,7 +222,7 @@ def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]:
     logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
 
     n = netnode.Netnode(CAPA_NETNODE)
-    doc = rdoc.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
+    doc = rdoc.ResultDocument.model_validate_json(n[NETNODE_RESULTS])
 
     for rule in rutils.capability_rules(doc):
         for location_, _ in rule.matches:

capa/main.py

@@ -1223,7 +1223,7 @@ def main(argv: Optional[List[str]] = None):
 
     if format_ == FORMAT_RESULT:
         # result document directly parses into meta, capabilities
-        result_doc = capa.render.result_document.ResultDocument.parse_file(args.sample)
+        result_doc = capa.render.result_document.ResultDocument.from_file(Path(args.sample))
         meta, capabilities = result_doc.to_capa()
 
     else:

capa/render/result_document.py

@@ -8,6 +8,7 @@
 import datetime
 import collections
 from typing import Dict, List, Tuple, Union, Literal, Optional
+from pathlib import Path
 
 from pydantic import Field, BaseModel, ConfigDict
 
@@ -596,3 +597,7 @@ def to_capa(self) -> Tuple[Metadata, Dict]:
                 capabilities[rule_name].append((addr.to_capa(), result))
 
         return self.meta, capabilities
+
+    @classmethod
+    def from_file(cls, path: Path) -> "ResultDocument":
+        return cls.model_validate_json(path.read_text(encoding="utf-8"))

pyproject.toml

@@ -77,10 +77,10 @@ dev = [
     "flake8-simplify==0.20.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.0.284",
+    "ruff==0.0.285",
     "black==23.7.0",
     "isort==5.11.4",
-    "mypy==1.5.0",
+    "mypy==1.5.1",
     "psutil==5.9.2",
     "stix2==3.0.1",
     "requests==2.31.0",

scripts/import-to-ida.py

@@ -30,6 +30,7 @@
 """
 import logging
 import binascii
+from pathlib import Path
 
 import ida_nalt
 import ida_funcs
@@ -68,7 +69,7 @@ def main():
     if not path:
         return 0
 
-    result_doc = capa.render.result_document.ResultDocument.parse_file(path)
+    result_doc = capa.render.result_document.ResultDocument.from_file(Path(path))
     meta, capabilities = result_doc.to_capa()
 
     # in IDA 7.4, the MD5 hash may be truncated, for example:

scripts/proto-from-results.py

@@ -31,6 +31,7 @@
 import sys
 import logging
 import argparse
+from pathlib import Path
 
 import capa.render.proto
 import capa.render.result_document
@@ -64,7 +65,7 @@ def main(argv=None):
         logging.basicConfig(level=logging.INFO)
         logging.getLogger().setLevel(logging.INFO)
 
-    rd = capa.render.result_document.ResultDocument.parse_file(args.json)
+    rd = capa.render.result_document.ResultDocument.from_file(Path(args.json))
     pb = capa.render.proto.doc_to_pb2(rd)
 
     sys.stdout.buffer.write(pb.SerializeToString(deterministic=True))

tests/fixtures.py

@@ -1182,8 +1182,8 @@ def _039a6_dotnetfile_extractor():
     return get_dnfile_extractor(get_data_path_by_name("_039a6"))
 
 
-def get_result_doc(path):
-    return capa.render.result_document.ResultDocument.parse_file(path)
+def get_result_doc(path: Path):
+    return capa.render.result_document.ResultDocument.from_file(path)
 
 
 @pytest.fixture

tests/test_ida_features.py

@@ -92,6 +92,15 @@ def get_ida_extractor(_path):
     return capa.features.extractors.ida.extractor.IdaFeatureExtractor()
 
 
+def nocollect(f):
+    "don't collect the decorated function as a pytest test"
+    f.__test__ = False
+    return f
+
+
+# although these look like pytest tests, they're not, because they don't run within pytest
+# (the runner is below) and they use `yield`, which is deprecated.
+@nocollect
 @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")
 def test_ida_features():
     # we're guaranteed to be in a function here, so there's a stack frame
@@ -118,6 +127,7 @@ def test_ida_features():
             yield this_name, id, "pass", None
 
 
+@nocollect
 @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")
 def test_ida_feature_counts():
     # we're guaranteed to be in a function here, so there's a stack frame

tests/test_result_document.py

@@ -237,7 +237,7 @@ def assert_round_trip(rd: rdoc.ResultDocument):
     one = rd
 
     doc = one.model_dump_json(exclude_none=True)
-    two = rdoc.ResultDocument.parse_raw(doc)
+    two = rdoc.ResultDocument.model_validate_json(doc)
 
     # show the round trip works
     # first by comparing the objects directly,
@@ -272,13 +272,13 @@ def test_round_trip(request, rd_file):
 
 def test_json_to_rdoc():
     path = fixtures.get_data_path_by_name("pma01-01-rd")
-    assert isinstance(rdoc.ResultDocument.parse_file(path), rdoc.ResultDocument)
+    assert isinstance(rdoc.ResultDocument.from_file(path), rdoc.ResultDocument)
 
 
 def test_rdoc_to_capa():
     path = fixtures.get_data_path_by_name("pma01-01-rd")
 
-    rd = rdoc.ResultDocument.parse_file(path)
+    rd = rdoc.ResultDocument.from_file(path)
 
     meta, capabilites = rd.to_capa()
     assert isinstance(meta, rdoc.Metadata)