eip7688: use forward compatible SSZ types in Gloas

EIP-4788 exposed the beacon root to smart contracts, but smart contracts
need to be redeployed / upgraded whenever generalized indices change
during a fork, even if that fork does not touch any used functionality.
That is analogous to an OS without ABI stability, requiring programs
to be maintained and re-compiled due to random breakages in OS updates.

This issue expands further to bridges on other blockchains, and also
into wallets / dApps that verify data from the beacon chain instead.
Such projects do not typically share Ethereum's release cadence.

- https://eips.ethereum.org/EIPS/eip-4788

EIP-7688 introduces forward compatibility for beacon chain structures.
Generalized indices remain same when list capacities evolve over forks,
containers no longer get re-indexed when reaching a new power-of-2
number of fields, and fields can be deprecated, leaving a gap in the
Merkle tree instead of triggering re-indexing.

- https://eips.ethereum.org/EIPS/eip-7688

EIP-7688 was requested for inclusion by popular projects:

- For Electra by Rocketpool: https://xcancel.com/KaneWallmann/status/1816729724145795258
- For Fulu by Lido: ethereum/pm#1356 (comment)

Author: etan-status

presets/mainnet/gloas.yaml

@@ -9,8 +9,3 @@ PTC_SIZE: 512
 # ---------------------------------------------------------------
 # 2**2 (= 4) attestations
 MAX_PAYLOAD_ATTESTATIONS: 4
-
-# State list lengths
-# ---------------------------------------------------------------
-# 2**20 (= 1,048,576) builder pending withdrawals
-BUILDER_PENDING_WITHDRAWALS_LIMIT: 1048576

presets/minimal/gloas.yaml

@@ -9,8 +9,3 @@ PTC_SIZE: 2
 # ---------------------------------------------------------------
 # 2**2 (= 4) attestations
 MAX_PAYLOAD_ATTESTATIONS: 4
-
-# State list lengths
-# ---------------------------------------------------------------
-# 2**20 (= 1,048,576) builder pending withdrawals
-BUILDER_PENDING_WITHDRAWALS_LIMIT: 1048576

pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
   "py_arkworks_bls12381==0.3.8",
   "py_ecc==8.0.0",
   "pycryptodome==3.23.0",
-  "remerkleable @ git+https://github.com/ethereum/remerkleable@92dcbb6f0507035d6875986fc263a05fec19d473",
+  "remerkleable @ git+https://github.com/ethereum/remerkleable@71a94389375aa9afbe39145dcd26d4cddafa140d",
   "ruamel.yaml==0.18.15",
   "setuptools==80.9.0",
   "trie==3.1.0",

pysetup/generate_specs.py

@@ -43,6 +43,7 @@
 from pysetup.helpers import (
     combine_spec_objects,
     dependency_order_class_objects,
+    finalized_spec_object,
     objects_to_spec,
     parse_config_vars,
 )
@@ -118,6 +119,7 @@ def build_spec(
     spec_object = all_specs[0]
     for value in all_specs[1:]:
         spec_object = combine_spec_objects(spec_object, value)
+    spec_object = finalized_spec_object(spec_object)
 
     class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses}
 
@@ -127,7 +129,7 @@ def build_spec(
         new_objects = copy.deepcopy(class_objects)
         dependency_order_class_objects(
             class_objects,
-            spec_object.custom_types | spec_object.preset_dep_custom_types,
+            spec_object.custom_types,
         )
 
     return objects_to_spec(preset_name, spec_object, fork, class_objects)

pysetup/helpers.py

@@ -23,8 +23,19 @@ def collect_prev_forks(fork: str) -> list[str]:
 
 
 def requires_mypy_type_ignore(value: str) -> bool:
-    return value.startswith("ByteVector") or (
-        value.startswith("Vector") and any(k in value for k in ["ceillog2", "floorlog2"])
+    return (
+        value.startswith("Bitlist")
+        or value.startswith("ByteVector")
+        or (value.startswith("List") and not re.match(r"^List\[\w+,\s*\w+\]$", value))
+        or (value.startswith("Vector") and any(k in value for k in ["ceillog2", "floorlog2"]))
+    )
+
+
+def gen_new_type_definition(name: str, value: str) -> str:
+    return (
+        f"class {name}({value}):\n    pass"
+        if not requires_mypy_type_ignore(value)
+        else f"class {name}(\n    {value}  # type: ignore\n):\n    pass"
     )
 
 
@@ -41,19 +52,11 @@ def objects_to_spec(
     """
 
     def gen_new_type_definitions(custom_types: dict[str, str]) -> str:
-        return "\n\n".join(
-            [
-                (
-                    f"class {key}({value}):\n    pass\n"
-                    if not requires_mypy_type_ignore(value)
-                    else f"class {key}({value}):  # type: ignore\n    pass\n"
-                )
-                for key, value in custom_types.items()
-            ]
+        return "\n\n\n".join(
+            [gen_new_type_definition(key, value) for key, value in custom_types.items()]
         )
 
     new_type_definitions = gen_new_type_definitions(spec_object.custom_types)
-    preset_dep_new_type_definitions = gen_new_type_definitions(spec_object.preset_dep_custom_types)
 
     # Collect builders with the reversed previous forks
     # e.g. `[bellatrix, altair, phase0]` -> `[phase0, altair, bellatrix]`
@@ -224,10 +227,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         ssz_dep_constants,
         new_type_definitions,
         constant_vars_spec,
-        # The presets that some SSZ types require. Need to be defined before `preset_dep_new_type_definitions`
+        # The presets that some SSZ types require.
         preset_vars_spec,
         preset_dep_constant_vars_spec,
-        preset_dep_new_type_definitions,
         config_spec,
         # Custom classes which are not required to be SSZ containers.
         classes,
@@ -267,8 +269,6 @@ def combine_dicts(old_dict: dict[str, T], new_dict: dict[str, T]) -> dict[str, T
     "bit",
     "Bitlist",
     "Bitvector",
-    "BLSPubkey",
-    "BLSSignature",
     "boolean",
     "byte",
     "ByteList",
@@ -292,6 +292,8 @@ def combine_dicts(old_dict: dict[str, T], new_dict: dict[str, T]) -> dict[str, T
     "floorlog2",
     "List",
     "Optional",
+    "ProgressiveBitlist",
+    "ProgressiveList",
     "Sequence",
     "Set",
     "Tuple",
@@ -312,10 +314,14 @@ def dependency_order_class_objects(objects: dict[str, str], custom_types: dict[s
     items = list(objects.items())
     for key, value in items:
         dependencies = []
-        for line in value.split("\n"):
-            if not re.match(r"\s+\w+: .+", line):
+        for i, line in enumerate(value.split("\n")):
+            if i == 0:
+                match = re.match(r".+\((.+)\):", line)
+            else:
+                match = re.match(r"\s+\w+: (.+)", line)
+            if not match:
                 continue  # skip whitespace etc.
-            line = line[line.index(":") + 1 :]  # strip of field name
+            line = match.group(1)
             if "#" in line:
                 line = line[: line.index("#")]  # strip of comment
             dependencies.extend(
@@ -349,9 +355,6 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     protocols = combine_protocols(spec0.protocols, spec1.protocols)
     functions = combine_dicts(spec0.functions, spec1.functions)
     custom_types = combine_dicts(spec0.custom_types, spec1.custom_types)
-    preset_dep_custom_types = combine_dicts(
-        spec0.preset_dep_custom_types, spec1.preset_dep_custom_types
-    )
     constant_vars = combine_dicts(spec0.constant_vars, spec1.constant_vars)
     preset_dep_constant_vars = combine_dicts(
         spec0.preset_dep_constant_vars, spec1.preset_dep_constant_vars
@@ -366,7 +369,6 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
         functions=functions,
         protocols=protocols,
         custom_types=custom_types,
-        preset_dep_custom_types=preset_dep_custom_types,
         constant_vars=constant_vars,
         preset_dep_constant_vars=preset_dep_constant_vars,
         preset_vars=preset_vars,
@@ -378,6 +380,41 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     )
 
 
+def finalized_spec_object(spec_object: SpecObject) -> SpecObject:
+    all_config_dependencies = {
+        vardef.type_name or vardef.type_hint
+        for vardef in (
+            spec_object.constant_vars
+            | spec_object.preset_dep_constant_vars
+            | spec_object.preset_vars
+            | spec_object.config_vars
+        ).values()
+        if (vardef.type_name or vardef.type_hint) is not None
+    }
+
+    custom_types = {}
+    ssz_objects = spec_object.ssz_objects
+    for name, value in spec_object.custom_types.items():
+        if any(k in name for k in all_config_dependencies):
+            custom_types[name] = value
+        else:
+            ssz_objects[name] = gen_new_type_definition(name, value)
+
+    return SpecObject(
+        functions=spec_object.functions,
+        protocols=spec_object.protocols,
+        custom_types=custom_types,
+        constant_vars=spec_object.constant_vars,
+        preset_dep_constant_vars=spec_object.preset_dep_constant_vars,
+        preset_vars=spec_object.preset_vars,
+        config_vars=spec_object.config_vars,
+        ssz_dep_constants=spec_object.ssz_dep_constants,
+        func_dep_presets=spec_object.func_dep_presets,
+        ssz_objects=ssz_objects,
+        dataclasses=spec_object.dataclasses,
+    )
+
+
 def parse_config_vars(conf: dict[str, str]) -> dict[str, str | list[dict[str, str]]]:
     """
     Parses a dict of basic str/int/list types into a dict for insertion into the spec code.

pysetup/md_to_spec.py

@@ -32,7 +32,6 @@ def __init__(
         self.preset_name = preset_name
 
         self.document_iterator: Iterator[Element] = self._parse_document(file_name)
-        self.all_custom_types: dict[str, str] = {}
         self.current_heading_name: str | None = None
 
         # Use a single dict to hold all SpecObject fields
@@ -44,7 +43,6 @@ def __init__(
             "func_dep_presets": {},
             "functions": {},
             "preset_dep_constant_vars": {},
-            "preset_dep_custom_types": {},
             "preset_vars": {},
             "protocols": {},
             "ssz_dep_constants": {},
@@ -180,8 +178,12 @@ def _process_code_class(self, source: str, cls: ast.ClassDef) -> None:
         if class_name != self.current_heading_name:
             raise Exception(f"class_name {class_name} != current_name {self.current_heading_name}")
 
-        if parent_class:
-            assert parent_class == "Container"
+        if parent_class == "ProgressiveContainer":
+            source = re.sub(
+                r"^(.*ProgressiveContainer.*)$", r"\1  # type: ignore", source, flags=re.MULTILINE
+            )
+        else:
+            assert parent_class is None or parent_class == "Container"
         self.spec["ssz_objects"][class_name] = source
 
     def _process_table(self, table: Table) -> None:
@@ -202,9 +204,21 @@ def _process_table(self, table: Table) -> None:
             if not _is_constant_id(name):
                 # Check for short type declarations
                 if value.startswith(
-                    ("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")
+                    (
+                        "uint",
+                        "Bitlist",
+                        "Bitvector",
+                        "ByteList",
+                        "ByteVector",
+                        "Bytes",
+                        "List",
+                        "ProgressiveBitlist",
+                        "ProgressiveList",
+                        "Union",
+                        "Vector",
+                    )
                 ):
-                    self.all_custom_types[name] = value
+                    self.spec["custom_types"][name] = value
                 continue
 
             # It is a constant name and a generalized index
@@ -418,7 +432,6 @@ def _process_html_block(self, html: HTMLBlock) -> None:
 
     def _finalize_types(self) -> None:
         """
-        Processes all_custom_types into custom_types and preset_dep_custom_types.
         Calls helper functions to update KZG and CURDLEPROOFS setups if needed.
         """
         # Update KZG trusted setup if needed
@@ -433,17 +446,6 @@ def _finalize_types(self) -> None:
                 self.spec["constant_vars"], self.preset_name
             )
 
-        # Split all_custom_types into custom_types and preset_dep_custom_types
-        self.spec["custom_types"] = {}
-        self.spec["preset_dep_custom_types"] = {}
-        for name, value in self.all_custom_types.items():
-            if any(k in value for k in self.preset) or any(
-                k in value for k in self.spec["preset_dep_constant_vars"]
-            ):
-                self.spec["preset_dep_custom_types"][name] = value
-            else:
-                self.spec["custom_types"][name] = value
-
     def _build_spec_object(self) -> SpecObject:
         """
         Returns the SpecObject using all collected data.
@@ -456,7 +458,6 @@ def _build_spec_object(self) -> SpecObject:
             func_dep_presets=self.spec["func_dep_presets"],
             functions=self.spec["functions"],
             preset_dep_constant_vars=self.spec["preset_dep_constant_vars"],
-            preset_dep_custom_types=self.spec["preset_dep_custom_types"],
             preset_vars=self.spec["preset_vars"],
             protocols=self.spec["protocols"],
             ssz_dep_constants=self.spec["ssz_dep_constants"],
@@ -496,6 +497,8 @@ def _get_class_info_from_ast(cls: ast.ClassDef) -> tuple[str, str | None]:
         parent_class = base.id
     elif isinstance(base, ast.Subscript):
         parent_class = base.value.id
+    elif isinstance(base, ast.Call):
+        parent_class = base.func.id
     else:
         # NOTE: SSZ definition derives from earlier phase...
         # e.g. `phase0.SignedBeaconBlock`

pysetup/spec_builders/gloas.py

@@ -8,6 +8,8 @@ class GloasSpecBuilder(BaseSpecBuilder):
     @classmethod
     def imports(cls, preset_name: str):
         return f"""
+from eth2spec.utils.ssz.ssz_typing import ProgressiveBitlist, ProgressiveContainer, ProgressiveList
+
 from eth2spec.fulu import {preset_name} as fulu
 """
 

pysetup/typing.py

@@ -18,7 +18,6 @@ class SpecObject(NamedTuple):
     functions: dict[str, str]
     protocols: dict[str, ProtocolDefinition]
     custom_types: dict[str, str]
-    preset_dep_custom_types: dict[str, str]  # the types that depend on presets
     constant_vars: dict[str, VariableDefinition]
     preset_dep_constant_vars: dict[str, VariableDefinition]
     preset_vars: dict[str, VariableDefinition]

specs/electra/beacon-chain.md

@@ -3,6 +3,7 @@
 <!-- mdformat-toc start --slug=github --no-anchors --maxlevel=6 --minlevel=2 -->
 
 - [Introduction](#introduction)
+- [Custom types](#custom-types)
 - [Constants](#constants)
   - [Misc](#misc)
   - [Withdrawal prefixes](#withdrawal-prefixes)
@@ -123,6 +124,16 @@ Electra is a consensus-layer upgrade containing a number of features. Including:
 *Note*: This specification is built upon [Deneb](../deneb/beacon-chain.md) and
 is under active development.
 
+## Custom types
+
+| Name                    | SSZ equivalent                                                                 | Description                                                        |
+| ----------------------- | ------------------------------------------------------------------------------ | ------------------------------------------------------------------ |
+| `AggregationBits`       | `Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]`              | Combined participation info across all participating subcommittees |
+| `AttestingIndices`      | `List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]` | List of attesting validator indices                                |
+| `DepositRequests`       | `List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD]`                       | List of deposit requests pertaining to an execution payload        |
+| `WithdrawalRequests`    | `List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD]`                 | List of withdrawal requests pertaining to an execution payload     |
+| `ConsolidationRequests` | `List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD]`           | List of withdrawal requests pertaining to an execution payload     |
+
 ## Constants
 
 The following values are (non-configurable) constants used throughout the
@@ -293,11 +304,11 @@ class ConsolidationRequest(Container):
 ```python
 class ExecutionRequests(Container):
     # [New in Electra:EIP6110]
-    deposits: List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD]
+    deposits: DepositRequests
     # [New in Electra:EIP7002:EIP7251]
-    withdrawals: List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD]
+    withdrawals: WithdrawalRequests
     # [New in Electra:EIP7251]
-    consolidations: List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD]
+    consolidations: ConsolidationRequests
 ```
 
 #### `SingleAttestation`
@@ -351,7 +362,7 @@ class BeaconBlockBody(Container):
 ```python
 class Attestation(Container):
     # [Modified in Electra:EIP7549]
-    aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]
+    aggregation_bits: AggregationBits
     data: AttestationData
     signature: BLSSignature
     # [New in Electra:EIP7549]
@@ -363,7 +374,7 @@ class Attestation(Container):
 ```python
 class IndexedAttestation(Container):
     # [Modified in Electra:EIP7549]
-    attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]
+    attesting_indices: AttestingIndices
     data: AttestationData
     signature: BLSSignature
 ```
@@ -1320,7 +1331,7 @@ def get_execution_requests_list(execution_requests: ExecutionRequests) -> Sequen
     return [
         request_type + ssz_serialize(request_data)
         for request_type, request_data in requests
-        if len(request_data) != 0
+        if request_data
     ]
 ```