Add support for newlines inside "NAME" token

dissect/cstruct/compiler.py

@@ -226,18 +226,18 @@ def align_to_field(field: Field) -> Iterator[str]:
 
     def _generate_structure(self, field: Field) -> Iterator[str]:
         template = f"""
-        {'_s = stream.tell()' if field.type.dynamic else ''}
+        {"_s = stream.tell()" if field.type.dynamic else ""}
         r["{field._name}"] = {self._map_field(field)}._read(stream, context=r)
-        {f's["{field._name}"] = stream.tell() - _s' if field.type.dynamic else ''}
+        {f's["{field._name}"] = stream.tell() - _s' if field.type.dynamic else ""}
         """
 
         yield dedent(template)
 
     def _generate_array(self, field: Field) -> Iterator[str]:
         template = f"""
-        {'_s = stream.tell()' if field.type.dynamic else ''}
+        {"_s = stream.tell()" if field.type.dynamic else ""}
         r["{field._name}"] = {self._map_field(field)}._read(stream, context=r)
-        {f's["{field._name}"] = stream.tell() - _s' if field.type.dynamic else ''}
+        {f's["{field._name}"] = stream.tell() - _s' if field.type.dynamic else ""}
         """
 
         yield dedent(template)

dissect/cstruct/cstruct.py

@@ -369,14 +369,14 @@ def _make_array(self, type_: T, num_entries: int | Expression | None) -> type[Ar
             "null_terminated": null_terminated,
         }
 
-        return cast(type[Array], self._make_type(name, bases, size, alignment=type_.alignment, attrs=attrs))
+        return cast("type[Array]", self._make_type(name, bases, size, alignment=type_.alignment, attrs=attrs))
 
     def _make_int_type(self, name: str, size: int, signed: bool, *, alignment: int | None = None) -> type[Int]:
-        return cast(type[Int], self._make_type(name, (Int,), size, alignment=alignment, attrs={"signed": signed}))
+        return cast("type[Int]", self._make_type(name, (Int,), size, alignment=alignment, attrs={"signed": signed}))
 
     def _make_packed_type(self, name: str, packchar: str, base: type, *, alignment: int | None = None) -> type[Packed]:
         return cast(
-            type[Packed],
+            "type[Packed]",
             self._make_type(
                 name,
                 (base, Packed),

dissect/cstruct/expression.py

@@ -141,7 +141,7 @@ def tokenize(self) -> list[str]:
                 self.tokens.append(">>")
             elif self.match(expected="<", append=False) and self.match(expected="<", append=False):
                 self.tokens.append("<<")
-            elif self.match(expected={" ", "\t"}, append=False):
+            elif self.match(expected={" ", "\n", "\t"}, append=False):
                 continue
             else:
                 raise ExpressionTokenizerError(

dissect/cstruct/parser.py

@@ -63,7 +63,7 @@ def _tokencollection() -> TokenCollection:
             "ENUM",
         )
         TOK.add(r"(?<=})\s*(?P<defs>(?:[a-zA-Z0-9_]+\s*,\s*)+[a-zA-Z0-9_]+)\s*(?=;)", "DEFS")
-        TOK.add(r"(?P<name>\**?\s*[a-zA-Z0-9_]+)(?:\s*:\s*(?P<bits>\d+))?(?:\[(?P<count>[^;\n]*)\])?\s*(?=;)", "NAME")
+        TOK.add(r"(?P<name>\**?\s*[a-zA-Z0-9_]+)(?:\s*:\s*(?P<bits>\d+))?(?:\[(?P<count>[^;]*)\])?\s*(?=;)", "NAME")
         TOK.add(r"#include\s+(?P<name>[^\s]+)\s*", "INCLUDE")
         TOK.add(r"[a-zA-Z_][a-zA-Z0-9_]*", "IDENTIFIER")
         TOK.add(r"[{}]", "BLOCK")
@@ -194,7 +194,7 @@ def _struct(self, tokens: TokenConsumer, register: bool = False) -> type[Structu
         if tokens.next == self.TOK.NAME:
             # As part of a struct field
             # struct type_name field_name;
-            if not len(names):
+            if not names:
                 raise ParserError(f"line {self._lineno(tokens.next)}: unexpected anonymous struct")
             return self.cstruct.resolve(names[0])
 

dissect/cstruct/types/structure.py

@@ -139,7 +139,7 @@ def _update_fields(
 
         if cls.__compiled__:
             # If the previous class was compiled try to compile this too
-            from dissect.cstruct import compiler
+            from dissect.cstruct import compiler  # noqa: PLC0415
 
             try:
                 classdict["_read"] = compiler.Compiler(cls.cs).compile_read(fields, cls.__name__, align=cls.__align__)

pyproject.toml

@@ -42,7 +42,7 @@ cstruct-stubgen = "dissect.cstruct.tools.stubgen:main"
 
 [tool.ruff]
 line-length = 120
-required-version = ">=0.9.0"
+required-version = ">=0.12.0"
 
 [tool.ruff.format]
 docstring-code-format = true
@@ -85,7 +85,7 @@ select = [
 ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003"]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/docs/**" = ["INP001"]
+"tests/_docs/**" = ["INP001"]
 
 [tool.ruff.lint.isort]
 known-first-party = ["dissect.cstruct"]

tests/test_annotations.py

@@ -21,7 +21,7 @@ def test_cstruct_type_annotation(name: str, monkeypatch: pytest.MonkeyPatch) ->
         for module in [module for module in sys.modules if module in ("dissect.cstruct.cstruct")]:
             monkeypatch.delitem(sys.modules, module)
 
-        from dissect.cstruct import cstruct
+        from dissect.cstruct import cstruct  # noqa: PLC0415
 
         if name.startswith("__"):
             name = f"_cstruct{name}"

tests/test_parser.py

@@ -149,4 +149,3 @@ def test_includes(cs: cstruct) -> None:
     assert cs.myStruct.__name__ == "myStruct"
     assert len(cs.myStruct.fields) == 1
     assert cs.myStruct.fields.get("charVal")
-

tests/test_types_structure.py

@@ -265,8 +265,8 @@ def test_structure_definition_simple(cs: cstruct, compiled: bool) -> None:
     with pytest.raises(AttributeError):
         obj.nope  # noqa: B018
 
-    assert obj.__dynamic_sizes__ == { "string": 7, "wstring": 10 }
-    assert obj.__sizes__ == { "magic": 4, "wmagic": 8, "a": 1, "b": 2, "c": 4, "string": 7, "wstring": 10 }
+    assert obj.__dynamic_sizes__ == {"string": 7, "wstring": 10}
+    assert obj.__sizes__ == {"magic": 4, "wmagic": 8, "a": 1, "b": 2, "c": 4, "string": 7, "wstring": 10}
     assert len(obj) == len(buf)
     assert obj.dumps() == buf
 
@@ -276,6 +276,7 @@ def test_structure_definition_simple(cs: cstruct, compiled: bool) -> None:
     obj.write(fh)
     assert fh.getvalue() == buf
 
+
 def test_structure_values_dict(cs: cstruct, compiled: bool) -> None:
     cdef = """
     struct test {
@@ -810,3 +811,36 @@ def test_codegen_hashable(cs: cstruct) -> None:
 
     assert hash(structure._generate_structure__init__(hashable_fields).__code__)
     assert hash(structure._generate_structure__init__(unhashable_fields).__code__)
+
+
+def test_structure_definition_newline(cs: cstruct, compiled: bool) -> None:
+    cdef = """
+    struct test {
+        char    magic[4
+        ];
+
+        wchar   wmagic[4];
+        uint8   a;
+        uint16  b;
+        uint32  c;
+        char    string[];
+        wchar   wstring[];
+    };
+    """
+    cs.endian = ">"
+    cs.load(cdef, compiled=compiled)
+
+    assert verify_compiled(cs.test, compiled)
+
+    buf = b"test\x00t\x00e\x00s\x00t\x01\x02\x03\x04\x05\x06\x07lalala\x00\x00t\x00e\x00s\x00t\x00\x00"
+
+    obj = cs.test()
+    obj.magic = b"test"
+    obj.wmagic = "test"
+    obj.a = 0x01
+    obj.b = 0x0203
+    obj.c = 0x04050607
+    obj.string = b"lalala"
+    obj.wstring = "test"
+
+    assert obj.dumps() == buf

tox.ini

@@ -32,16 +32,18 @@ commands =
 [testenv:fix]
 package = skip
 deps =
-    ruff==0.9.2
+    ruff==0.12.4
 commands =
     ruff format dissect tests
+    ruff check --fix dissect tests
 
 [testenv:lint]
 package = skip
 deps =
-    ruff==0.9.2
+    ruff==0.12.4
     vermin
 commands =
+    ruff format --check dissect tests
     ruff check dissect tests
     vermin -t=3.9- --no-tips --lint dissect tests