refactor: refines msgpack decoding in api generator and boolean fields in block model

Improves msgpack decoding in Algod, Indexer and KMD clients by handling byte keys and values. This prevents decoding errors when encountering non-UTF-8 byte sequences.

Additionally, adds decoding for boolean fields in block models to correctly interpret raw values as booleans. This addresses issues with inconsistent data representation.

Author: aorumbayev

api/oas-generator/src/oas_generator/builder.py

@@ -694,4 +694,5 @@ def build_client_descriptor(
         uses_signed_transaction=uses_signed_txn,
         uses_msgpack=operation_builder.uses_msgpack,
         include_block_models=operation_builder.uses_block_models,
+        include_ledger_state_delta_models="LedgerStateDelta" in registry.entries,
     )

api/oas-generator/src/oas_generator/models.py

@@ -146,3 +146,4 @@ class ClientDescriptor:
     uses_signed_transaction: bool = False
     uses_msgpack: bool = False
     include_block_models: bool = False
+    include_ledger_state_delta_models: bool = False

api/oas-generator/src/oas_generator/renderer/engine.py

@@ -26,6 +26,31 @@ class TemplateRenderer:
         "Block",
         "GetBlock",
     ]
+    LEDGER_STATE_DELTA_EXPORTS: ClassVar[list[str]] = [
+        "LedgerTealValue",
+        "LedgerStateSchema",
+        "LedgerAppParams",
+        "LedgerAppLocalState",
+        "LedgerAppLocalStateDelta",
+        "LedgerAppParamsDelta",
+        "LedgerAppResourceRecord",
+        "LedgerAssetHolding",
+        "LedgerAssetHoldingDelta",
+        "LedgerAssetParams",
+        "LedgerAssetParamsDelta",
+        "LedgerAssetResourceRecord",
+        "LedgerVotingData",
+        "LedgerAccountBaseData",
+        "LedgerAccountData",
+        "LedgerBalanceRecord",
+        "LedgerAccountDeltas",
+        "LedgerKvValueDelta",
+        "LedgerIncludedTransactions",
+        "LedgerModifiedCreatable",
+        "LedgerAlgoCount",
+        "LedgerAccountTotals",
+        "LedgerStateDelta",
+    ]
 
     def __init__(self, template_dir: Path | None = None) -> None:
         if template_dir:
@@ -56,6 +81,8 @@ def render(self, client: ctx.ClientDescriptor, config: GeneratorConfig) -> dict[
         files[models_dir / "__init__.py"] = self._render_template("models/__init__.py.j2", context)
         files[models_dir / "_serde_helpers.py"] = self._render_template("models/_serde_helpers.py.j2", context)
         for model in context["client"].models:
+            if context["client"].include_ledger_state_delta_models and model.name == "LedgerStateDelta":
+                continue
             model_context = {**context, "model": model}
             files[models_dir / f"{model.module_name}.py"] = self._render_template("models/model.py.j2", model_context)
         for enum in context["client"].enums:
@@ -67,7 +94,11 @@ def render(self, client: ctx.ClientDescriptor, config: GeneratorConfig) -> dict[
                 "models/type_alias.py.j2", alias_context
             )
         if client.include_block_models:
-            files[models_dir / "block.py"] = self._render_template("models/block.py.j2", context)
+            files[models_dir / "_block.py"] = self._render_template("models/block.py.j2", context)
+        if client.include_ledger_state_delta_models:
+            files[models_dir / "_ledger_state_delta.py"] = self._render_template(
+                "models/ledger_state_delta.py.j2", context
+            )
         files[target / "py.typed"] = ""
         return files
 
@@ -85,6 +116,10 @@ def _build_context(self, client: ctx.ClientDescriptor, config: GeneratorConfig)
             for name in self.BLOCK_MODEL_EXPORTS:
                 if name not in model_exports:
                     model_exports.append(name)
+        if client.include_ledger_state_delta_models:
+            for name in self.LEDGER_STATE_DELTA_EXPORTS:
+                if name not in model_exports:
+                    model_exports.append(name)
         metadata_usage = self._collect_metadata_usage(client)
         model_modules = [{"module": model.module_name, "name": model.name} for model in client.models]
         enum_modules = [{"module": enum.module_name, "name": enum.name} for enum in client.enums]
@@ -105,6 +140,7 @@ def _build_context(self, client: ctx.ClientDescriptor, config: GeneratorConfig)
             "needs_datetime": any(model.requires_datetime for model in client.models),
             "client_needs_datetime": self._client_requires_datetime(client),
             "block_exports": self.BLOCK_MODEL_EXPORTS,
+            "ledger_state_delta_exports": self.LEDGER_STATE_DELTA_EXPORTS,
             "needs_literal": needs_literal,
         }
 

api/oas-generator/src/oas_generator/renderer/templates/client.py.j2

@@ -248,7 +248,7 @@ class {{ client.class_name }}:
             return response.content
         content_type = response.headers.get("content-type", "application/json")
         if "msgpack" in content_type:
-            data = msgpack.unpackb(response.content, raw=False, strict_map_key=False)
+            data = msgpack.unpackb(response.content, raw=True, strict_map_key=False)
             data = self._normalize_msgpack(data)
         elif content_type.startswith("application/json"):
             data = response.json()
@@ -264,7 +264,18 @@ class {{ client.class_name }}:
 
     def _normalize_msgpack(self, value: object) -> object:
         if isinstance(value, dict):
-            return {key: self._normalize_msgpack(item) for key, item in value.items()}
+            normalized: dict[object, object] = {}
+            for key, item in value.items():
+                normalized[self._coerce_msgpack_key(key)] = self._normalize_msgpack(item)
+            return normalized
         if isinstance(value, list):
             return [self._normalize_msgpack(item) for item in value]
         return value
+
+    def _coerce_msgpack_key(self, key: object) -> object:
+        if isinstance(key, bytes):
+            try:
+                return key.decode("utf-8")
+            except UnicodeDecodeError:
+                return key
+        return key

api/oas-generator/src/oas_generator/renderer/templates/models/__init__.py.j2

@@ -1,16 +1,18 @@
 
 
 {% if client.uses_signed_transaction %}from algokit_transact.models.signed_transaction import SignedTransaction
-{% endif %}{% for item in model_modules %}from .{{ item.module }} import {{ item.name }}
-{% endfor %}{% for item in enum_modules %}from .{{ item.module }} import {{ item.name }}
+{% endif %}{% for item in model_modules %}{% if not (client.include_ledger_state_delta_models and item.name == "LedgerStateDelta") %}from .{{ item.module }} import {{ item.name }}
+{% endif %}{% endfor %}{% for item in enum_modules %}from .{{ item.module }} import {{ item.name }}
 {% endfor %}{% for item in alias_modules %}from .{{ item.module }} import {{ item.name }}
-{% endfor %}{% if client.include_block_models %}from .block import (
+{% endfor %}{% if client.include_block_models %}from ._block import (
     {{ block_exports | join(',\n    ') }}
 )
+{% endif %}{% if client.include_ledger_state_delta_models %}from ._ledger_state_delta import (
+    {{ ledger_state_delta_exports | join(',\n    ') }}
+)
 {% endif %}
 
 __all__ = [
     {% for name in model_exports %}"{{ name }}",
     {% endfor %}
 ]
-

api/oas-generator/src/oas_generator/renderer/templates/models/_serde_helpers.py.j2

@@ -8,9 +8,9 @@ from typing import Callable, TypeAlias, TypeVar
 
 from algokit_common.serde import from_wire, to_wire
 
-T = TypeVar("T")
-E = TypeVar("E", bound=Enum)
-KT = TypeVar("KT")
+DecodedT = TypeVar("DecodedT")
+EnumValueT = TypeVar("EnumValueT", bound=Enum)
+MapKeyT = TypeVar("MapKeyT")
 BytesLike: TypeAlias = bytes | bytearray | memoryview
 
 
@@ -33,11 +33,22 @@ def decode_bytes_base64(raw: object) -> bytes:
         try:
             return base64.b64decode(raw.encode("ascii"), validate=True)
         except (BinasciiError, UnicodeEncodeError) as exc:
+            raise ValueError("Invalid base64 payload") from exc
+    raise TypeError(f"Unsupported value for bytes field: {type(raw)!r}")
+
+
+def decode_bytes_map_key(raw: object) -> bytes:
+    if isinstance(raw, bytes | bytearray | memoryview):
+        return bytes(raw)
+    if isinstance(raw, str):
+        try:
+            return decode_bytes_base64(raw)
+        except ValueError:
             try:
                 return raw.encode("utf-8")
             except UnicodeEncodeError as fallback_exc:
-                raise ValueError("Invalid base64 payload") from fallback_exc
-    raise TypeError(f"Unsupported value for bytes field: {type(raw)!r}")
+                raise ValueError("Invalid bytes map key") from fallback_exc
+    raise TypeError(f"Unsupported map key for bytes field: {type(raw)!r}")
 
 
 def encode_bytes_sequence(values: Iterable[BytesLike | None] | None) -> list[str | None] | None:
@@ -77,11 +88,11 @@ def encode_model_sequence(values: Iterable[object] | None) -> list[dict[str, obj
     return encoded or None
 
 
-def decode_model_sequence(cls_factory: Callable[[], type[T]], raw: object) -> list[T] | None:
+def decode_model_sequence(cls_factory: Callable[[], type[DecodedT]], raw: object) -> list[DecodedT] | None:
     if not isinstance(raw, list):
         return None
     cls = cls_factory()
-    decoded: list[T] = []
+    decoded: list[DecodedT] = []
     for item in raw:
         if isinstance(item, Mapping):
             decoded.append(from_wire(cls, item))
@@ -99,11 +110,11 @@ def encode_enum_sequence(values: Iterable[object] | None) -> list[object] | None
     return encoded or None
 
 
-def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> list[E] | None:
+def decode_enum_sequence(enum_factory: Callable[[], type[EnumValueT]], raw: object) -> list[EnumValueT] | None:
     if not isinstance(raw, list):
         return None
     enum_cls = enum_factory()
-    decoded: list[E] = []
+    decoded: list[EnumValueT] = []
     for item in raw:
         try:
             decoded.append(enum_cls(item))
@@ -113,7 +124,7 @@ def decode_enum_sequence(enum_factory: Callable[[], type[E]], raw: object) -> li
 
 
 def encode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     mapping: Mapping[object, object] | None,
     *,
     key_encoder: Callable[[object], str] | None = None,
@@ -140,24 +151,30 @@ def encode_model_mapping(
 
 
 def decode_model_mapping(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     raw: object,
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> dict[MapKeyT, DecodedT] | None:
     if not isinstance(raw, Mapping):
         return None
     cls = factory()
-    decoded: dict[KT, T] = {}
+    decoded: dict[MapKeyT, DecodedT] = {}
     for key, value in raw.items():
         if isinstance(value, Mapping):
             decoded_key = key_decoder(key) if key_decoder is not None else key
             decoded[decoded_key] = from_wire(cls, value)
     return decoded or None
 
 
+def decode_optional_bool(raw: object) -> bool | None:
+    if raw is None:
+        return None
+    return bool(raw)
+
+
 def mapping_encoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
     key_encoder: Callable[[object], str] | None = None,
 ) -> Callable[[Mapping[object, object] | None], dict[str, object] | None]:
@@ -168,11 +185,11 @@ def mapping_encoder(
 
 
 def mapping_decoder(
-    factory: Callable[[], type[T]],
+    factory: Callable[[], type[DecodedT]],
     *,
-    key_decoder: Callable[[object], KT] | None = None,
-) -> Callable[[object], dict[KT, T] | None]:
-    def _decode(raw: object) -> dict[KT, T] | None:
+    key_decoder: Callable[[object], MapKeyT] | None = None,
+) -> Callable[[object], dict[MapKeyT, DecodedT] | None]:
+    def _decode(raw: object) -> dict[MapKeyT, DecodedT] | None:
         return decode_model_mapping(factory, raw, key_decoder=key_decoder)
 
     return _decode

api/oas-generator/src/oas_generator/renderer/templates/models/block.py.j2

@@ -4,13 +4,14 @@ from collections.abc import Mapping
 from dataclasses import dataclass, field
 from typing import Any, cast
 
-from algokit_common.serde import flatten, nested, wire
+from algokit_common.serde import flatten, nested, wire, addr
 from algokit_transact.models.signed_transaction import SignedTransaction
 
 from ._serde_helpers import (
-    decode_bytes_base64,
+    decode_bytes_map_key,
     decode_model_mapping,
     decode_model_sequence,
+    decode_optional_bool,
     encode_bytes_base64,
     encode_model_mapping,
     encode_model_sequence,
@@ -64,18 +65,28 @@ def _decode_state_proof_tracking_key(key: object) -> int:
 
 
 def _decode_block_state_delta(raw: object) -> BlockStateDelta | None:
-    decoded = decode_model_mapping(lambda: BlockEvalDelta, raw, key_decoder=decode_bytes_base64)
+    decoded = decode_model_mapping(lambda: BlockEvalDelta, raw, key_decoder=decode_bytes_map_key)
     return decoded or None
 
 
+def _encode_local_delta_index_key(key: object) -> str:
+    if isinstance(key, bool):
+        return str(int(key))
+    if isinstance(key, int):
+        return str(key)
+    if isinstance(key, str):
+        return str(int(key))
+    raise TypeError("Local delta keys must be numeric")
+
+
 def _encode_local_deltas(mapping: Mapping[int, BlockStateDelta] | None) -> dict[str, object] | None:
     if mapping is None:
         return None
     out: dict[str, object] = {}
     for key, value in mapping.items():
         encoded = _encode_block_state_delta(value)
         if encoded:
-            out[str(int(key))] = encoded
+            out[_encode_local_delta_index_key(key)] = encoded
     return out or None
 
 
@@ -156,8 +167,8 @@ class SignedTxnInBlock:
     )
     config_asset: int | None = field(default=None, metadata=wire("caid"))
     application_id: int | None = field(default=None, metadata=wire("apid"))
-    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi"))
-    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh"))
+    has_genesis_id: bool | None = field(default=None, metadata=wire("hgi", decode=decode_optional_bool))
+    has_genesis_hash: bool | None = field(default=None, metadata=wire("hgh", decode=decode_optional_bool))
 
 
 @dataclass(slots=True)
@@ -206,12 +217,12 @@ class Block:
     timestamp: int | None = field(default=None, metadata=wire("ts"))
     genesis_id: str | None = field(default=None, metadata=wire("gen"))
     genesis_hash: bytes | None = field(default=None, metadata=wire("gh"))
-    proposer: bytes | None = field(default=None, metadata=wire("prp"))
+    proposer: bytes | None = field(default=None, metadata=addr("prp"))
     fees_collected: int | None = field(default=None, metadata=wire("fc"))
     bonus: int | None = field(default=None, metadata=wire("bi"))
     proposer_payout: int | None = field(default=None, metadata=wire("pp"))
-    fee_sink: bytes | None = field(default=None, metadata=wire("fees"))
-    rewards_pool: bytes | None = field(default=None, metadata=wire("rwd"))
+    fee_sink: bytes | None = field(default=None, metadata=addr("fees"))
+    rewards_pool: bytes | None = field(default=None, metadata=addr("rwd"))
     rewards_level: int | None = field(default=None, metadata=wire("earn"))
     rewards_rate: int | None = field(default=None, metadata=wire("rate"))
     rewards_residue: int | None = field(default=None, metadata=wire("frac"))