refactor: refines msgpack decoding in api generator and boolean fields in block model

Improves msgpack decoding in Algod, Indexer and KMD clients by handling byte keys and values. This prevents decoding errors when encountering non-UTF-8 byte sequences.

Additionally, adds decoding for boolean fields in block models to correctly interpret raw values as booleans. This addresses issues with inconsistent data representation.

Author: aorumbayev

api/oas-generator/src/oas_generator/renderer/templates/client.py.j2

@@ -248,7 +248,7 @@ class {{ client.class_name }}:
             return response.content
         content_type = response.headers.get("content-type", "application/json")
         if "msgpack" in content_type:
-            data = msgpack.unpackb(response.content, raw=False, strict_map_key=False)
+            data = msgpack.unpackb(response.content, raw=True, strict_map_key=False)
             data = self._normalize_msgpack(data)
         elif content_type.startswith("application/json"):
             data = response.json()
@@ -264,7 +264,23 @@ class {{ client.class_name }}:
 
     def _normalize_msgpack(self, value: object) -> object:
         if isinstance(value, dict):
-            return {key: self._normalize_msgpack(item) for key, item in value.items()}
+            normalized: dict[object, object] = {}
+            for key, item in value.items():
+                normalized[self._coerce_msgpack_key(key)] = self._normalize_msgpack(item)
+            return normalized
         if isinstance(value, list):
             return [self._normalize_msgpack(item) for item in value]
+        if isinstance(value, bytes):
+            try:
+                return value.decode("utf-8")
+            except UnicodeDecodeError:
+                return value
         return value
+
+    def _coerce_msgpack_key(self, key: object) -> object:
+        if isinstance(key, bytes):
+            try:
+                return key.decode("utf-8")
+            except UnicodeDecodeError:
+                return key
+        return key

api/oas-generator/src/oas_generator/renderer/templates/models/_serde_helpers.py.j2

@@ -8,9 +8,9 @@ from typing import Callable, TypeAlias, TypeVar
 
 from algokit_common.serde import from_wire, to_wire
 
-T = TypeVar("T")
-E = TypeVar("E", bound=Enum)
-KT = TypeVar("KT")
+DecodedT = TypeVar("DecodedT")
+EnumValueT = TypeVar("EnumValueT", bound=Enum)
+MapKeyT = TypeVar("MapKeyT")
 BytesLike: TypeAlias = bytes | bytearray | memoryview
 
 
@@ -77,11 +77,11 @@ def encode_model_sequence(values: Iterable[object] | None) -> list[dict[str, obj
     return encoded or None
 
 
-def decode_model_sequence(cls_factory: Callable[[], type[T]], raw: object) -> list[T] | None:
+def decode_model_sequence(cls_factory: Callable[[], type[DecodedT]], raw: object) -> list[DecodedT] | None:
     if not isinstance(raw, list):
         return None
     cls = cls_factory()
-    decoded: list[T] = []
+    decoded: list[DecodedT] = []
     for item in raw:
         if isinstance(item, Mapping):
             decoded.append(from_wire(cls, item))
@@ -99,11 +99,11 @@ def encode_enum_sequence(values: Iterable[object] | None) -> list[object] | None
     return encoded or None
 
 
-def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> list[E] | None:
+def decode_enum_sequence(enum_factory: Callable[[], type[EnumValueT]], raw: object) -> list[EnumValueT] | None:
     if not isinstance(raw, list):
         return None
     enum_cls = enum_factory()
-    decoded: list[E] = []
+    decoded: list[EnumValueT] = []
     for item in raw:
         try:
             decoded.append(enum_cls(item))
@@ -113,7 +113,7 @@ def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> li
 
 
 def encode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     mapping: Mapping[object, object] | None,
     *,
     key_encoder: Callable[[object], str] | None = None,
@@ -140,15 +140,15 @@ def encode_model_mapping(
 
 
 def decode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     raw: object,
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> dict[MapKeyT, DecodedT] | None:
     if not isinstance(raw, Mapping):
         return None
     cls = factory()
-    decoded: dict[KT, T] = {}
+    decoded: dict[MapKeyT, DecodedT] = {}
     for key, value in raw.items():
         if isinstance(value, Mapping):
             decoded_key = key_decoder(key) if key_decoder is not None else key
@@ -157,7 +157,7 @@ def decode_model_mapping(
 
 
 def mapping_encoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
     key_encoder: Callable[[object], str] | None = None,
 ) -> Callable[[Mapping[object, object] | None], dict[str, object] | None]:
@@ -168,11 +168,11 @@ def mapping_encoder(
 
 
 def mapping_decoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> Callable[[object], dict[KT, T] | None]:
-    def _decode(raw: object) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> Callable[[object], dict[MapKeyT, DecodedT] | None]:
+    def _decode(raw: object) -> dict[MapKeyT, DecodedT] | None:
         return decode_model_mapping(factory, raw, key_decoder=key_decoder)
 
     return _decode

api/oas-generator/src/oas_generator/renderer/templates/models/block.py.j2

@@ -156,8 +156,8 @@ class SignedTxnInBlock:
     )
     config_asset: int | None = field(default=None, metadata=wire("caid"))
     application_id: int | None = field(default=None, metadata=wire("apid"))
-    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi"))
-    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh"))
+    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi", decode=lambda raw: bool(raw)))
+    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh", decode=lambda raw: bool(raw)))
 
 
 @dataclass(slots=True)

api/specs/compatibility.md

@@ -1937,7 +1937,7 @@ def _decode_response(
             return response.content
         content_type = response.headers.get("content-type", "application/json")
         if "msgpack" in content_type:
-            data = msgpack.unpackb(response.content, raw=False, strict_map_key=False)
+            data = msgpack.unpackb(response.content, raw=True, strict_map_key=False)
             data = self._normalize_msgpack(data)
         elif content_type.startswith("application/json"):
             data = response.json()
@@ -1953,7 +1953,23 @@ def _decode_response(
 
     def _normalize_msgpack(self, value: object) -> object:
         if isinstance(value, dict):
-            return {key: self._normalize_msgpack(item) for key, item in value.items()}
+            normalized: dict[object, object] = {}
+            for key, item in value.items():
+                normalized[self._coerce_msgpack_key(key)] = self._normalize_msgpack(item)
+            return normalized
         if isinstance(value, list):
             return [self._normalize_msgpack(item) for item in value]
+        if isinstance(value, bytes):
+            try:
+                return value.decode("utf-8")
+            except UnicodeDecodeError:
+                return value
         return value
+
+    def _coerce_msgpack_key(self, key: object) -> object:
+        if isinstance(key, bytes):
+            try:
+                return key.decode("utf-8")
+            except UnicodeDecodeError:
+                return key
+        return key