refactor: refines msgpack decoding in api generator and boolean fields in block model

Improves msgpack decoding in Algod, Indexer and KMD clients by handling byte keys and values. This prevents decoding errors when encountering non-UTF-8 byte sequences.

Additionally, adds decoding for boolean fields in block models to correctly interpret raw values as booleans. This addresses issues with inconsistent data representation.

Author: aorumbayev

api/oas-generator/src/oas_generator/renderer/templates/client.py.j2

@@ -248,7 +248,7 @@ class {{ client.class_name }}:
             return response.content
         content_type = response.headers.get("content-type", "application/json")
         if "msgpack" in content_type:
-            data = msgpack.unpackb(response.content, raw=False, strict_map_key=False)
+            data = msgpack.unpackb(response.content, raw=True, strict_map_key=False)
             data = self._normalize_msgpack(data)
         elif content_type.startswith("application/json"):
             data = response.json()
@@ -264,7 +264,23 @@ class {{ client.class_name }}:
 
     def _normalize_msgpack(self, value: object) -> object:
         if isinstance(value, dict):
-            return {key: self._normalize_msgpack(item) for key, item in value.items()}
+            normalized: dict[object, object] = {}
+            for key, item in value.items():
+                normalized[self._coerce_msgpack_key(key)] = self._normalize_msgpack(item)
+            return normalized
         if isinstance(value, list):
             return [self._normalize_msgpack(item) for item in value]
+        if isinstance(value, bytes):
+            try:
+                return value.decode("utf-8")
+            except UnicodeDecodeError:
+                return value
         return value
+
+    def _coerce_msgpack_key(self, key: object) -> object:
+        if isinstance(key, bytes):
+            try:
+                return key.decode("utf-8")
+            except UnicodeDecodeError:
+                return key
+        return key

api/oas-generator/src/oas_generator/renderer/templates/models/_serde_helpers.py.j2

@@ -8,9 +8,9 @@ from typing import Callable, TypeAlias, TypeVar
 
 from algokit_common.serde import from_wire, to_wire
 
-T = TypeVar("T")
-E = TypeVar("E", bound=Enum)
-KT = TypeVar("KT")
+DecodedT = TypeVar("DecodedT")
+EnumValueT = TypeVar("EnumValueT", bound=Enum)
+MapKeyT = TypeVar("MapKeyT")
 BytesLike: TypeAlias = bytes | bytearray | memoryview
 
 
@@ -32,7 +32,7 @@ def decode_bytes_base64(raw: object) -> bytes:
     if isinstance(raw, str):
         try:
             return base64.b64decode(raw.encode("ascii"), validate=True)
-        except (BinasciiError, UnicodeEncodeError) as exc:
+        except (BinasciiError, UnicodeEncodeError):
             try:
                 return raw.encode("utf-8")
             except UnicodeEncodeError as fallback_exc:
@@ -77,11 +77,11 @@ def encode_model_sequence(values: Iterable[object] | None) -> list[dict[str, obj
     return encoded or None
 
 
-def decode_model_sequence(cls_factory: Callable[[], type[T]], raw: object) -> list[T] | None:
+def decode_model_sequence(cls_factory: Callable[[], type[DecodedT]], raw: object) -> list[DecodedT] | None:
     if not isinstance(raw, list):
         return None
     cls = cls_factory()
-    decoded: list[T] = []
+    decoded: list[DecodedT] = []
     for item in raw:
         if isinstance(item, Mapping):
             decoded.append(from_wire(cls, item))
@@ -99,11 +99,11 @@ def encode_enum_sequence(values: Iterable[object] | None) -> list[object] | None
     return encoded or None
 
 
-def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> list[E] | None:
+def decode_enum_sequence(enum_factory: Callable[[], type[EnumValueT]], raw: object) -> list[EnumValueT] | None:
     if not isinstance(raw, list):
         return None
     enum_cls = enum_factory()
-    decoded: list[E] = []
+    decoded: list[EnumValueT] = []
     for item in raw:
         try:
             decoded.append(enum_cls(item))
@@ -113,7 +113,7 @@ def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> li
 
 
 def encode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     mapping: Mapping[object, object] | None,
     *,
     key_encoder: Callable[[object], str] | None = None,
@@ -140,24 +140,30 @@ def encode_model_mapping(
 
 
 def decode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     raw: object,
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> dict[MapKeyT, DecodedT] | None:
     if not isinstance(raw, Mapping):
         return None
     cls = factory()
-    decoded: dict[KT, T] = {}
+    decoded: dict[MapKeyT, DecodedT] = {}
     for key, value in raw.items():
         if isinstance(value, Mapping):
             decoded_key = key_decoder(key) if key_decoder is not None else key
             decoded[decoded_key] = from_wire(cls, value)
     return decoded or None
 
 
+def decode_optional_bool(raw: object) -> bool | None:
+    if raw is None:
+        return None
+    return bool(raw)
+
+
 def mapping_encoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
     key_encoder: Callable[[object], str] | None = None,
 ) -> Callable[[Mapping[object, object] | None], dict[str, object] | None]:
@@ -168,11 +174,11 @@ def mapping_encoder(
 
 
 def mapping_decoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> Callable[[object], dict[KT, T] | None]:
-    def _decode(raw: object) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> Callable[[object], dict[MapKeyT, DecodedT] | None]:
+    def _decode(raw: object) -> dict[MapKeyT, DecodedT] | None:
         return decode_model_mapping(factory, raw, key_decoder=key_decoder)
 
     return _decode

api/oas-generator/src/oas_generator/renderer/templates/models/block.py.j2

@@ -11,6 +11,7 @@ from ._serde_helpers import (
     decode_bytes_base64,
     decode_model_mapping,
     decode_model_sequence,
+    decode_optional_bool,
     encode_bytes_base64,
     encode_model_mapping,
     encode_model_sequence,
@@ -68,14 +69,22 @@ def _decode_block_state_delta(raw: object) -> BlockStateDelta | None:
     return decoded or None
 
 
+def _encode_local_delta_index_key(key: object) -> str:
+    if isinstance(key, bool):
+        return str(int(key))
+    if isinstance(key, int):
+        return str(key)
+    return str(int(key))
+
+
 def _encode_local_deltas(mapping: Mapping[int, BlockStateDelta] | None) -> dict[str, object] | None:
     if mapping is None:
         return None
     out: dict[str, object] = {}
     for key, value in mapping.items():
         encoded = _encode_block_state_delta(value)
         if encoded:
-            out[str(int(key))] = encoded
+            out[_encode_local_delta_index_key(key)] = encoded
     return out or None
 
 
@@ -156,8 +165,8 @@ class SignedTxnInBlock:
     )
     config_asset: int | None = field(default=None, metadata=wire("caid"))
     application_id: int | None = field(default=None, metadata=wire("apid"))
-    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi"))
-    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh"))
+    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi", decode=decode_optional_bool))
+    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh", decode=decode_optional_bool))
 
 
 @dataclass(slots=True)

api/specs/compatibility.md

@@ -1937,7 +1937,7 @@ def _decode_response(
             return response.content
         content_type = response.headers.get("content-type", "application/json")
         if "msgpack" in content_type:
-            data = msgpack.unpackb(response.content, raw=False, strict_map_key=False)
+            data = msgpack.unpackb(response.content, raw=True, strict_map_key=False)
             data = self._normalize_msgpack(data)
         elif content_type.startswith("application/json"):
             data = response.json()
@@ -1953,7 +1953,23 @@ def _decode_response(
 
     def _normalize_msgpack(self, value: object) -> object:
         if isinstance(value, dict):
-            return {key: self._normalize_msgpack(item) for key, item in value.items()}
+            normalized: dict[object, object] = {}
+            for key, item in value.items():
+                normalized[self._coerce_msgpack_key(key)] = self._normalize_msgpack(item)
+            return normalized
         if isinstance(value, list):
             return [self._normalize_msgpack(item) for item in value]
+        if isinstance(value, bytes):
+            try:
+                return value.decode("utf-8")
+            except UnicodeDecodeError:
+                return value
         return value
+
+    def _coerce_msgpack_key(self, key: object) -> object:
+        if isinstance(key, bytes):
+            try:
+                return key.decode("utf-8")
+            except UnicodeDecodeError:
+                return key
+        return key