fix: operation and sanity spec tests v1.5.0

.github/workflows/CI.yml

@@ -90,3 +90,33 @@ jobs:
       - name: Run Int Tests
         run: |
           zig build test:int
+  spec-test:
+    name: spec test
+    # It takes time to download and run spec tests, so we only run them on ubuntu-latest
+    runs-on: ubuntu-latest
+    needs: build-test
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Install Zig
+        uses: mlugg/setup-zig@v2
+        with:
+          version: ${{ env.ZIG_VERSION }}
+          cache-key: ${{ matrix.os }}-${{ env.ZIG_VERSION }}
+      - name: Restore spec tests cache
+        uses: actions/cache@v4
+        with:
+          path: test/spec/spec_tests
+          key: spec-test-data-${{ hashFiles('build.zig') }} # spec test version is defined in build.zig
+      - name: Download spec tests
+        run: |
+          zig build run:download_spec_tests
+      - name: Write spec tests
+        run: |
+          zig build run:write_spec_tests
+      - name: Run spec tests - minimal
+        run: |
+          zig build test:spec_tests -Dpreset=minimal
+      - name: Run spec tests - mainnet
+        run: |
+          zig build test:spec_tests -Dpreset=mainnet

src/config/beacon_config.zig

@@ -213,7 +213,7 @@ pub const BeaconConfig = struct {
     }
 
     pub fn getDomainForVoluntaryExit(self: *const BeaconConfig, state_slot: Slot, message_slot: ?Slot) ![32]u8 {
-        const domain = if (state_slot < self.chain.DENEB_FORK_EPOCH * preset.SLOTS_PER_EPOCH) {
+        const domain = if (state_slot / preset.SLOTS_PER_EPOCH < self.chain.DENEB_FORK_EPOCH) {
             return self.getDomain(state_slot, DOMAIN_VOLUNTARY_EXIT, message_slot);
         } else {
             return self.getDomainByForkSeq(ForkSeq.capella, DOMAIN_VOLUNTARY_EXIT);
@@ -227,8 +227,12 @@ pub const BeaconConfig = struct {
 };
 
 fn computeDomain(domain_type: DomainType, fork_version: Version, genesis_validators_root: Root, out: *[32]u8) !void {
-    try computeForkDataRoot(fork_version, genesis_validators_root, out);
-    std.mem.copyForwards(u8, out[0..], domain_type[0..]);
+    var fork_data_root: [32]u8 = undefined;
+    try computeForkDataRoot(fork_version, genesis_validators_root, &fork_data_root);
+    // 4 first bytes is domain_type
+    std.mem.copyForwards(u8, out[0..4], domain_type[0..4]);
+    // 28 next bytes is first 28 bytes of fork_data_root
+    std.mem.copyForwards(u8, out[4..], fork_data_root[0..28]);
 }
 
 fn computeForkDataRoot(current_version: Version, genesis_validators_root: Root, out: *[32]u8) !void {

src/config/chain/chain_config.zig

@@ -1,3 +1,4 @@
+const std = @import("std");
 const ssz = @import("consensus_types");
 const Epoch = ssz.primitive.Epoch.Type;
 const Preset = @import("preset").Preset;
@@ -91,3 +92,20 @@ pub const BlobScheduleEntry = struct {
     EPOCH: Epoch,
     MAX_BLOBS_PER_BLOCK: u64,
 };
+
+pub fn mergeChainConfig(config: ChainConfig, fields: anytype) ChainConfig {
+    var merged = config;
+    inline for (std.meta.fields(@TypeOf(fields))) |field| {
+        @field(merged, field.name) = @field(fields, field.name);
+    }
+    return merged;
+}
+
+test mergeChainConfig {
+    const mainnet_config = @import("./networks/mainnet.zig").mainnet_chain_config;
+    const old_altair_epoch = mainnet_config.ALTAIR_FORK_EPOCH;
+    const merged_config = mergeChainConfig(mainnet_config, .{
+        .ALTAIR_FORK_EPOCH = old_altair_epoch + 1000,
+    });
+    try std.testing.expect(merged_config.ALTAIR_FORK_EPOCH == old_altair_epoch + 1000);
+}

src/config/root.zig

@@ -5,6 +5,7 @@ pub const ChainConfig = @import("./chain/chain_config.zig").ChainConfig;
 pub const ForkSeq = @import("./fork.zig").ForkSeq;
 pub const ForkInfo = @import("./fork.zig").ForkInfo;
 pub const forkSeqByForkName = @import("./fork.zig").forkSeqByForkName;
+pub const mergeChainConfig = @import("./chain/chain_config.zig").mergeChainConfig;
 pub const TOTAL_FORKS = @import("./fork.zig").TOTAL_FORKS;
 pub const mainnet_chain_config = @import("./chain/networks/mainnet.zig").mainnet_chain_config;
 pub const minimal_chain_config = @import("./chain/networks/minimal.zig").minimal_chain_config;

src/constants/root.zig

@@ -5,6 +5,7 @@ pub const DEPOSIT_CONTRACT_TREE_DEPTH = std.math.pow(usize, 2, 5); // 32
 pub const JUSTIFICATION_BITS_LENGTH = 4;
 pub const ZERO_HASH = [_]u8{0} ** 32;
 pub const ZERO_HASH_HEX = "0x0000000000000000000000000000000000000000000000000000000000000000";
+pub const GENESIS_SLOT = 0;
 
 // Withdrawal prefixes
 // Since the prefixes are just 1 byte, we define and use them as number
@@ -108,7 +109,6 @@ pub const NEXT_SYNC_COMMITTEE_INDEX_ELECTRA = 23;
 pub const DEPOSIT_REQUEST_TYPE = 0x00;
 pub const WITHDRAWAL_REQUEST_TYPE = 0x01;
 pub const CONSOLIDATION_REQUEST_TYPE = 0x02;
-pub const PENDING_PARTIAL_WITHDRAWALS_LIMIT = 134_217_728;
 
 pub const CURRENT_SYNC_COMMITTEE_GINDEX = 54;
 pub const EXECUTION_PAYLOAD_GINDEX = 25;

src/preset/preset.zig

@@ -82,7 +82,6 @@ const PresetMainnet = struct {
     pub const MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 8;
     pub const DEPOSIT_CONTRACT_TREE_DEPTH = 32;
     pub const GENESIS_SLOT = 0;
-    pub const FAR_FUTURE_EPOCH = 18_446_744_073_709_551_615; // 2*64 -1;
     pub const MAX_PENDING_DEPOSITS_PER_EPOCH = 16;
 };
 
@@ -151,6 +150,7 @@ const PresetMinimal = struct {
     pub const MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD = 16;
     pub const MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD = 2;
     pub const WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA = 4096;
+    pub const MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA = 4096;
     pub const FIELD_ELEMENTS_PER_CELL = 64;
     pub const FIELD_ELEMENTS_PER_EXT_BLOB = 8192;
     pub const KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH = 4;

src/state_transition/block/is_valid_indexed_attestation.zig

@@ -15,6 +15,7 @@ pub fn isValidIndexedAttestation(comptime IA: type, cached_state: *const CachedB
 
     if (verify_signature) {
         const signature_set = try getIndexedAttestationSignatureSet(IA, cached_state.allocator, cached_state, indexed_attestation);
+        defer cached_state.allocator.free(signature_set.pubkeys);
         return try verifyAggregatedSignatureSet(&signature_set);
     } else {
         return true;
@@ -37,8 +38,10 @@ pub fn isValidIndexedAttestationIndices(cached_state: *const CachedBeaconStateAl
     // Just check if they are monotonically increasing,
     // instead of creating a set and sorting it. Should be (O(n)) instead of O(n log(n))
     var prev: ValidatorIndex = 0;
-    for (indices) |index| {
-        if (index <= prev) return false;
+    for (indices, 0..) |index, i| {
+        if (i >= 1 and index <= prev) {
+            return false;
+        }
         prev = index;
     }
 

src/state_transition/block/process_attestation_altair.zig

@@ -25,10 +25,11 @@ const TIMELY_TARGET = 1 << c.TIMELY_TARGET_FLAG_INDEX;
 const TIMELY_HEAD = 1 << c.TIMELY_HEAD_FLAG_INDEX;
 const SLOTS_PER_EPOCH_SQRT = std.math.sqrt(preset.SLOTS_PER_EPOCH);
 
-/// AT = AttestationType
-/// for phase0 it's `ssz.phase0.Attestation.Type`
-/// for electra it's `ssz.electra.Attestation.Type`
-pub fn processAttestationsAltair(allocator: Allocator, cached_state: *const CachedBeaconStateAllForks, comptime AT: type, attestations: []AT, verify_signature: bool) !void {
+pub fn processAttestationsAltair(allocator: Allocator, cached_state: *const CachedBeaconStateAllForks, attestations: anytype, verify_signature: bool) !void {
+    // AT = AttestationType
+    // for phase0 it's `ssz.phase0.Attestation.Type`
+    // for electra it's `ssz.electra.Attestation.Type`
+    const AT = @typeInfo(@TypeOf(attestations)).pointer.child;
     const state = cached_state.state;
     const epoch_cache = cached_state.getEpochCache();
     const effective_balance_increments = epoch_cache.effective_balance_increment.get().items;
@@ -44,19 +45,20 @@ pub fn processAttestationsAltair(allocator: Allocator, cached_state: *const Cach
     // let newSeenAttestersEffectiveBalance = 0;
 
     var proposer_reward: u64 = 0;
-    for (attestations) |attestation| {
+    for (attestations) |*attestation| {
         const data = attestation.data;
-        try validateAttestation(AT, cached_state, attestation);
+        try validateAttestation(cached_state, attestation);
 
         // Retrieve the validator indices from the attestation participation bitfield
-        const attesting_indices = try if (AT == Phase0Attestation) epoch_cache.getAttestingIndicesPhase0(&attestation) else epoch_cache.getAttestingIndicesElectra(&attestation);
+        const attesting_indices = try if (AT == Phase0Attestation) epoch_cache.getAttestingIndicesPhase0(attestation) else epoch_cache.getAttestingIndicesElectra(attestation);
         defer attesting_indices.deinit();
 
         // this check is done last because its the most expensive (if signature verification is toggled on)
         // TODO: Why should we verify an indexed attestation that we just created? If it's just for the signature
         // we can verify only that and nothing else.
         if (verify_signature) {
             const sig_set = try getAttestationWithIndicesSignatureSet(allocator, cached_state, &attestation.data, attestation.signature, attesting_indices.items);
+            defer allocator.free(sig_set.pubkeys);
             if (!try verifyAggregatedSignatureSet(&sig_set)) {
                 return error.InvalidSignature;
             }
@@ -77,8 +79,8 @@ pub fn processAttestationsAltair(allocator: Allocator, cached_state: *const Cach
             // At epoch boundary, 100% of attestations belong to previous epoch
             // so we want to update the participation flag tree in batch
 
-            // Note ParticipationFlags type uses option {setBitwiseOR: true}, .set() does a |= operation
-            epoch_participation[validator_index] = flags_attestation;
+            // no setBitwiseOR implemented in zig ssz, so we do it manually here
+            epoch_participation[validator_index] = flags_attestation | flags;
 
             // Returns flags that are NOT set before (~ bitwise NOT) AND are set after
             const flags_new_set = ~flags & flags_attestation;
@@ -107,15 +109,13 @@ pub fn processAttestationsAltair(allocator: Allocator, cached_state: *const Cach
                     }
                 }
             }
-
-            // Do the discrete math inside the loop to ensure a deterministic result
-            const total_increments = total_balance_increments_with_weight;
-            const proposer_reward_numerator = total_increments * epoch_cache.base_reward_per_increment;
-            proposer_reward += @divFloor(proposer_reward_numerator, PROPOSER_REWARD_DOMINATOR);
         }
-
-        increaseBalance(state, try epoch_cache.getBeaconProposer(state_slot), proposer_reward);
+        // Do the discrete math inside the loop to ensure a deterministic result
+        const total_increments = total_balance_increments_with_weight;
+        const proposer_reward_numerator = total_increments * epoch_cache.base_reward_per_increment;
+        proposer_reward += @divFloor(proposer_reward_numerator, PROPOSER_REWARD_DOMINATOR);
     }
+    increaseBalance(state, try epoch_cache.getBeaconProposer(state_slot), proposer_reward);
 }
 
 pub fn getAttestationParticipationStatus(state: *const BeaconStateAllForks, data: ssz.phase0.AttestationData.Type, inclusion_delay: u64, current_epoch: Epoch, root_cache: *RootCache) !u8 {

src/state_transition/block/process_attestation_phase0.zig

@@ -3,6 +3,7 @@ const Allocator = std.mem.Allocator;
 const CachedBeaconStateAllForks = @import("../cache/state_cache.zig").CachedBeaconStateAllForks;
 const BeaconStateAllForks = @import("../types/beacon_state.zig").BeaconStateAllForks;
 const ssz = @import("consensus_types");
+const s = @import("ssz");
 const preset = @import("preset").preset;
 const ForkSeq = @import("config").ForkSeq;
 const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot;
@@ -19,11 +20,21 @@ pub fn processAttestationPhase0(allocator: Allocator, cached_state: *CachedBeaco
     const slot = state.slot();
     const data = attestation.data;
 
-    try validateAttestation(*const Phase0Attestation, cached_state, attestation);
+    try validateAttestation(cached_state, attestation);
+
+    // should store a clone of aggregation_bits on Phase0 BeaconState to avoid double free error
+    var cloned_aggregation_bits: s.BitListType(preset.MAX_VALIDATORS_PER_COMMITTEE).Type = undefined;
+    try s.BitListType(preset.MAX_VALIDATORS_PER_COMMITTEE).clone(allocator, &attestation.aggregation_bits, &cloned_aggregation_bits);
+    var appended: bool = false;
+    errdefer {
+        if (!appended) {
+            cloned_aggregation_bits.deinit(allocator);
+        }
+    }
 
     const pending_attestation = PendingAttestation{
         .data = data,
-        .aggregation_bits = attestation.aggregation_bits,
+        .aggregation_bits = cloned_aggregation_bits,
         .inclusion_delay = slot - data.slot,
         .proposer_index = try epoch_cache.getBeaconProposer(slot),
     };
@@ -39,18 +50,25 @@ pub fn processAttestationPhase0(allocator: Allocator, cached_state: *CachedBeaco
         }
         try state.previousEpochPendingAttestations().append(allocator, pending_attestation);
     }
-    const indexed_attestation = try epoch_cache.getIndexedAttestation(.{
-        .phase0 = attestation.*,
-    });
+    appended = true;
 
-    _ = try isValidIndexedAttestation(ssz.phase0.IndexedAttestation.Type, cached_state, indexed_attestation.phase0, verify_signature);
+    var indexed_attestation: ssz.phase0.IndexedAttestation.Type = undefined;
+    try epoch_cache.computeIndexedAttestationPhase0(attestation, &indexed_attestation);
+    defer indexed_attestation.attesting_indices.deinit(allocator);
+
+    if (!try isValidIndexedAttestation(ssz.phase0.IndexedAttestation.Type, cached_state, &indexed_attestation, verify_signature)) {
+        return error.InvalidAttestationInvalidIndexedAttestation;
+    }
 }
 
 /// AT could be either Phase0Attestation or ElectraAttestation
-pub fn validateAttestation(comptime AT: type, cached_state: *const CachedBeaconStateAllForks, attestation: AT) !void {
+pub fn validateAttestation(cached_state: *const CachedBeaconStateAllForks, attestation: anytype) !void {
+    const T = @typeInfo(@TypeOf(attestation)).pointer.child;
+    std.debug.assert(T == Phase0Attestation or T == ElectraAttestation);
+    const is_electra = T == ElectraAttestation;
     const epoch_cache = cached_state.getEpochCache();
     const state = cached_state.state;
-    const slot = state.slot();
+    const state_slot = state.slot();
     const data = attestation.data;
     const computed_epoch = computeEpochAtSlot(data.slot);
     const committee_count = try epoch_cache.getCommitteeCountPerSlot(computed_epoch);
@@ -64,12 +82,12 @@ pub fn validateAttestation(comptime AT: type, cached_state: *const CachedBeaconS
     }
 
     // post deneb, the attestations are valid till end of next epoch
-    if (!(data.slot + preset.MIN_ATTESTATION_INCLUSION_DELAY <= slot and isTimelyTarget(state, slot - data.slot))) {
+    if (!(data.slot + preset.MIN_ATTESTATION_INCLUSION_DELAY <= state_slot and isTimelyTarget(state, state_slot - data.slot))) {
         return error.InvalidAttestationSlotNotWithInInclusionWindow;
     }
 
     // same to fork >= ForkSeq.electra but more type safe
-    if (AT == ElectraAttestation) {
+    if (is_electra) {
         if (data.index != 0) {
             return error.InvalidAttestationNonZeroDataIndex;
         }
@@ -93,7 +111,7 @@ pub fn validateAttestation(comptime AT: type, cached_state: *const CachedBeaconS
         // instead of implementing/calling getBeaconCommittees(slot, committee_indices.items), we call getBeaconCommittee(slot, index)
         var committee_offset: usize = 0;
         for (committee_indices) |committee_index| {
-            const committee_validators = try epoch_cache.getBeaconCommittee(slot, committee_index);
+            const committee_validators = try epoch_cache.getBeaconCommittee(data.slot, committee_index);
             if (committee_offset + committee_validators.len > aggregation_bits_array.len) {
                 return error.InvalidAttestationCommitteeAggregationBitsLengthTooShort;
             }
@@ -123,7 +141,7 @@ pub fn validateAttestation(comptime AT: type, cached_state: *const CachedBeaconS
             return error.InvalidAttestationInvalidCommitteeIndex;
         }
 
-        const committee = try epoch_cache.getBeaconCommittee(slot, data.index);
+        const committee = try epoch_cache.getBeaconCommittee(data.slot, data.index);
         if (attestation.aggregation_bits.bit_len != committee.len) {
             return error.InvalidAttestationInvalidAggregationBitLen;
         }

src/state_transition/block/process_attestations.zig

@@ -18,7 +18,7 @@ pub fn processAttestations(allocator: Allocator, cached_state: *CachedBeaconStat
         .phase0 => |attestations_phase0| {
             if (state.isPostAltair()) {
                 // altair to deneb
-                try processAttestationsAltair(allocator, cached_state, ssz.phase0.Attestation.Type, attestations_phase0.items, verify_signatures);
+                try processAttestationsAltair(allocator, cached_state, attestations_phase0.items, verify_signatures);
             } else {
                 // phase0
                 for (attestations_phase0.items) |attestation| {
@@ -27,7 +27,7 @@ pub fn processAttestations(allocator: Allocator, cached_state: *CachedBeaconStat
             }
         },
         .electra => |attestations_electra| {
-            try processAttestationsAltair(allocator, cached_state, ssz.electra.Attestation.Type, attestations_electra.items, verify_signatures);
+            try processAttestationsAltair(allocator, cached_state, attestations_electra.items, verify_signatures);
         },
     }
 }