feat: impl merkle_proof spec test

build.zig

@@ -326,4 +326,5 @@ pub fn build(b: *std.Build) void {
     module_spec_tests.addImport("state_transition", module_state_transition);
     module_spec_tests.addImport("ssz", dep_ssz.module("ssz"));
     module_spec_tests.addImport("blst", dep_blst.module("blst"));
+    module_spec_tests.addImport("persistent_merkle_tree", dep_ssz.module("persistent_merkle_tree"));
 }

test/spec/root.zig

@@ -4,6 +4,7 @@
 const testing = @import("std").testing;
 
 comptime {
+    testing.refAllDecls(@import("./test_case/merkle_proof_tests.zig"));
     testing.refAllDecls(@import("./test_case/operations_tests.zig"));
     testing.refAllDecls(@import("./test_case/sanity_tests.zig"));
 }

test/spec/runner/merkle_proof.zig

@@ -0,0 +1,206 @@
+const std = @import("std");
+const ct = @import("consensus_types");
+const ForkSeq = @import("config").ForkSeq;
+const preset_mod = @import("preset");
+const test_case = @import("../test_case.zig");
+const loadSszValue = test_case.loadSszSnappyValue;
+
+const Root = ct.primitive.Root.Type;
+const pmt = @import("persistent_merkle_tree");
+const Node = pmt.Node;
+const NodeId = Node.Id;
+const Gindex = pmt.Gindex;
+
+pub const Handler = enum {
+    single_merkle_proof,
+
+    pub fn suiteName(self: Handler) []const u8 {
+        return @tagName(self);
+    }
+};
+
+const MerkleProof = struct {
+    leaf: Root,
+    leaf_index: u64,
+    branch: []Root,
+
+    pub fn deinit(self: *MerkleProof, allocator: std.mem.Allocator) void {
+        allocator.free(self.branch);
+    }
+};
+
+pub fn TestCase(comptime fork: ForkSeq, comptime handler: Handler) type {
+    _ = handler;
+    const ForkTypes = @field(ct, fork.forkName());
+    const BeaconBlockBody = ForkTypes.BeaconBlockBody;
+    const KzgCommitment = ct.primitive.KZGCommitment;
+
+    return struct {
+        body: BeaconBlockBody.Type,
+        proof: MerkleProof,
+
+        const Self = @This();
+
+        pub fn execute(allocator: std.mem.Allocator, dir: std.fs.Dir) !void {
+            var tc = try Self.init(allocator, dir);
+            defer tc.deinit(allocator);
+
+            try tc.runTest(allocator);
+        }
+
+        fn init(allocator: std.mem.Allocator, dir: std.fs.Dir) !Self {
+            var body = BeaconBlockBody.default_value;
+            errdefer {
+                if (comptime @hasDecl(BeaconBlockBody, "deinit")) {
+                    BeaconBlockBody.deinit(allocator, &body);
+                }
+            }
+            try loadSszValue(BeaconBlockBody, allocator, dir, "object.ssz_snappy", &body);
+
+            const proof = try loadProof(allocator, dir);
+            errdefer proof.deinit(allocator);
+
+            return .{
+                .body = body,
+                .proof = proof,
+            };
+        }
+
+        fn deinit(self: *Self, allocator: std.mem.Allocator) void {
+            self.proof.deinit(allocator);
+            if (comptime @hasDecl(BeaconBlockBody, "deinit")) {
+                BeaconBlockBody.deinit(allocator, &self.body);
+            }
+        }
+
+        fn runTest(self: *Self, allocator: std.mem.Allocator) !void {
+            try verifyLeaf(self);
+            try verifyBranch(allocator, self);
+        }
+
+        fn verifyLeaf(self: *Self) !void {
+            // TODO: handle post-Fulu forks where blob_kzg_commitments is a list root, similar to Lodestar merkleProof tests.
+            const leaf_gindex_value = preset_mod.KZG_COMMITMENT_GINDEX0;
+            const actual_leaf_index: u64 = @intCast(leaf_gindex_value);
+
+            try std.testing.expectEqual(self.proof.leaf_index, actual_leaf_index);
+
+            if (self.body.blob_kzg_commitments.items.len == 0) {
+                return error.EmptyBlobKzgCommitments;
+            }
+
+            var actual_leaf: Root = undefined;
+            try KzgCommitment.hashTreeRoot(&self.body.blob_kzg_commitments.items[0], &actual_leaf);
+
+            try std.testing.expect(std.mem.eql(u8, &self.proof.leaf, &actual_leaf));
+        }
+
+        fn verifyBranch(allocator: std.mem.Allocator, self: *Self) !void {
+            var arena = std.heap.ArenaAllocator.init(allocator);
+            defer arena.deinit();
+            const arena_allocator = arena.allocator();
+
+            var pool = try Node.Pool.init(allocator, 2048);
+            defer pool.deinit();
+
+            const root_node = try BeaconBlockBody.tree.fromValue(arena_allocator, &pool, &self.body);
+
+            var actual_branch: std.ArrayListUnmanaged(Root) = .empty;
+            defer actual_branch.deinit(allocator);
+
+            try buildBranch(&pool, root_node, preset_mod.KZG_COMMITMENT_GINDEX0, allocator, &actual_branch);
+
+            try std.testing.expectEqualSlices(Root, self.proof.branch, actual_branch.items);
+        }
+
+        fn buildBranch(
+            pool: *Node.Pool,
+            root_node: Node.Id,
+            leaf_gindex_value: usize,
+            allocator: std.mem.Allocator,
+            branch_out: *std.ArrayListUnmanaged(Root),
+        ) !void {
+            // TODO: switch to a persistent_merkle_tree helper if/when one exists (e.g. getSingleProof).
+            const leaf_gindex = Gindex.fromUint(@as(Gindex.Uint, @intCast(leaf_gindex_value)));
+
+            var current = leaf_gindex;
+            while (@intFromEnum(current) > 1) {
+                const sibling = if ((@intFromEnum(current) & 1) == 0)
+                    @as(Gindex, @enumFromInt(@intFromEnum(current) + 1))
+                else
+                    @as(Gindex, @enumFromInt(@intFromEnum(current) - 1));
+
+                const sibling_node = try NodeId.getNode(root_node, pool, sibling);
+                const sibling_root = sibling_node.getRoot(pool);
+                try branch_out.append(allocator, sibling_root.*);
+
+                current = @as(Gindex, @enumFromInt(@intFromEnum(current) >> 1));
+            }
+        }
+
+        fn loadProof(allocator: std.mem.Allocator, dir: std.fs.Dir) !MerkleProof {
+            var file = try dir.openFile("proof.yaml", .{});
+            defer file.close();
+
+            const contents = try file.readToEndAlloc(allocator, 4096);
+            defer allocator.free(contents);
+
+            return parseProofYaml(allocator, contents);
+        }
+
+        fn parseProofYaml(allocator: std.mem.Allocator, contents: []const u8) !MerkleProof {
+            var proof = MerkleProof{
+                .leaf = undefined,
+                .leaf_index = 0,
+                .branch = &.{},
+            };
+
+            var branch: std.ArrayListUnmanaged(Root) = .empty;
+            errdefer branch.deinit(allocator);
+
+            var leaf_parsed = false;
+            var index_parsed = false;
+
+            var iter = std.mem.tokenizeScalar(u8, contents, '\n');
+            while (iter.next()) |line| {
+                const trimmed = std.mem.trim(u8, line, " \r\t");
+                if (trimmed.len == 0) continue;
+
+                if (std.mem.startsWith(u8, trimmed, "leaf:")) {
+                    const value_slice = std.mem.trim(u8, trimmed["leaf:".len..], " \t");
+                    proof.leaf = try parseHexRoot(value_slice);
+                    leaf_parsed = true;
+                } else if (std.mem.startsWith(u8, trimmed, "leaf_index:")) {
+                    const value_slice = std.mem.trim(u8, trimmed["leaf_index:".len..], " \t");
+                    proof.leaf_index = try std.fmt.parseInt(u64, value_slice, 10);
+                    index_parsed = true;
+                } else if (trimmed[0] == '-') {
+                    const value_slice = std.mem.trim(u8, trimmed[1..], " '\t");
+                    const branch_value = try parseHexRoot(value_slice);
+                    try branch.append(allocator, branch_value);
+                }
+            }
+
+            if (!leaf_parsed or !index_parsed) {
+                return error.InvalidProof;
+            }
+
+            proof.branch = try branch.toOwnedSlice(allocator);
+            return proof;
+        }
+
+        fn parseHexRoot(raw_value: []const u8) !Root {
+            var value = std.mem.trim(u8, raw_value, " '\t\"");
+            if (std.mem.startsWith(u8, value, "0x")) {
+                value = value[2..];
+            }
+            if (value.len != 64) {
+                return error.InvalidHexLength;
+            }
+
+            var out: Root = undefined;
+            _ = try std.fmt.hexToBytes(out[0..], value);
+            return out;
+        }
+    };
+}

test/spec/runner_kind.zig

@@ -3,9 +3,17 @@ const std = @import("std");
 pub const RunnerKind = enum {
     epoch_processing,
     finality,
+    merkle_proof,
     operations,
     random,
     rewards,
     sanity,
     shuffling,
+
+    pub fn hasSuiteCase(comptime self: RunnerKind) bool {
+        return switch (self) {
+            .merkle_proof => true,
+            else => false,
+        };
+    }
 };

test/spec/write_spec_tests.zig

@@ -13,12 +13,14 @@ const supported_forks = [_]ForkSeq{
 };
 
 const supported_test_runners = [_]RunnerKind{
+    .merkle_proof,
     .operations,
     .sanity,
 };
 
 fn TestWriter(comptime kind: RunnerKind) type {
     return switch (kind) {
+        .merkle_proof => @import("./writer/merkle_proof.zig"),
         .operations => @import("./writer/operations.zig"),
         .sanity => @import("./writer/sanity.zig"),
         else => @compileError("Unsupported test runner"),
@@ -27,10 +29,7 @@ fn TestWriter(comptime kind: RunnerKind) type {
 
 pub fn main() !void {
     const test_case_dir = "test/spec/test_case/";
-    std.fs.cwd().makeDir(test_case_dir) catch |err| {
-        if (err != error.PathAlreadyExists) return err;
-        // ignore if the directory already exists
-    };
+    try std.fs.cwd().makePath(test_case_dir);
 
     inline for (supported_test_runners) |kind| {
         const test_case_file = test_case_dir ++ @tagName(kind) ++ "_tests.zig";
@@ -43,6 +42,7 @@ pub fn main() !void {
 
     {
         const test_root_file = "test/spec/root.zig";
+        try std.fs.cwd().makePath("test/spec");
         const out = try std.fs.cwd().createFile(test_root_file, .{});
         defer out.close();
         const writer = out.writer().any();
@@ -87,22 +87,39 @@ pub fn writeTests(
     defer preset_dir.close();
 
     inline for (forks) |fork| {
-        var fork_dir = try preset_dir.openDir(@tagName(fork) ++ "/" ++ @tagName(kind), .{});
-        defer fork_dir.close();
+        const fork_path = @tagName(fork) ++ "/" ++ @tagName(kind);
+        const maybe_fork_dir = preset_dir.openDir(fork_path, .{ .iterate = true }) catch |err| switch (err) {
+            error.FileNotFound => null,
+            else => return err,
+        };
+
+        if (maybe_fork_dir) |dir| {
+            var fork_dir = dir;
+            defer fork_dir.close();
 
-        inline for (TestWriter(kind).handlers) |handler| {
-            st: {
-                var suite_dir = fork_dir.openDir(comptime handler.suiteName(), .{ .iterate = true }) catch break :st;
+            inline for (TestWriter(kind).handlers) |handler| handler_loop: {
+                var suite_dir = fork_dir.openDir(comptime handler.suiteName(), .{ .iterate = true }) catch |err| switch (err) {
+                    error.FileNotFound => break :handler_loop,
+                    else => return err,
+                };
                 defer suite_dir.close();
 
-                var test_case_iterator = suite_dir.iterate();
-                while (try test_case_iterator.next()) |test_case_entry| {
-                    if (test_case_entry.kind != .directory) {
-                        continue;
-                    }
-                    const test_case_name = test_case_entry.name;
+                var suite_iter = suite_dir.iterate();
+                while (try suite_iter.next()) |suite_entry| {
+                    if (suite_entry.kind != .directory) continue;
 
-                    try TestWriter(kind).writeTest(writer, fork, handler, test_case_name);
+                    if (comptime kind.hasSuiteCase()) {
+                        var case_dir = suite_dir.openDir(suite_entry.name, .{ .iterate = true }) catch continue;
+                        defer case_dir.close();
+
+                        var case_iter = case_dir.iterate();
+                        while (try case_iter.next()) |case_entry| {
+                            if (case_entry.kind != .directory) continue;
+                            try TestWriter(kind).writeTest(writer, fork, handler, suite_entry.name, case_entry.name);
+                        }
+                    } else {
+                        try TestWriter(kind).writeTest(writer, fork, handler, suite_entry.name);
+                    }
                 }
             }
         }

test/spec/writer/merkle_proof.zig

@@ -0,0 +1,63 @@
+const std = @import("std");
+const spec_test_options = @import("spec_test_options");
+const ForkSeq = @import("config").ForkSeq;
+const MerkleProof = @import("../runner/merkle_proof.zig");
+
+pub const handlers = std.enums.values(MerkleProof.Handler);
+pub const header =
+    \\// This file is generated by write_spec_tests.zig.
+    \\// Do not commit changes by hand.
+    \\
+    \\const std = @import("std");
+    \\const ForkSeq = @import("config").ForkSeq;
+    \\const active_preset = @import("preset").active_preset;
+    \\const spec_test_options = @import("spec_test_options");
+    \\const MerkleProof = @import("../runner/merkle_proof.zig");
+    \\
+    \\const allocator = std.testing.allocator;
+    \\
+    \\
+;
+
+const test_template =
+    \\test "{s} merkle_proof {s} {s} {s}" {{
+    \\    const test_dir_name = try std.fs.path.join(allocator, &[_][]const u8{{
+    \\        spec_test_options.spec_test_out_dir,
+    \\        spec_test_options.spec_test_version,
+    \\        @tagName(active_preset) ++ "/tests/" ++ @tagName(active_preset) ++ "/{s}/merkle_proof/{s}/{s}/{s}",
+    \\    }});
+    \\    defer allocator.free(test_dir_name);
+    \\    const test_dir = std.fs.cwd().openDir(test_dir_name, .{{}}) catch return error.SkipZigTest;
+    \\
+    \\    try MerkleProof.TestCase(.{s}, .{s}).execute(allocator, test_dir);
+    \\}}
+    \\
+    \\
+;
+
+pub fn writeHeader(writer: std.io.AnyWriter) !void {
+    try writer.print(header, .{});
+}
+
+pub fn writeTest(
+    writer: std.io.AnyWriter,
+    fork: ForkSeq,
+    handler: MerkleProof.Handler,
+    test_suite_name: []const u8,
+    test_case_name: []const u8,
+) !void {
+    try writer.print(test_template, .{
+        @tagName(fork),
+        @tagName(handler),
+        test_suite_name,
+        test_case_name,
+
+        @tagName(fork),
+        @tagName(handler),
+        test_suite_name,
+        test_case_name,
+
+        @tagName(fork),
+        @tagName(handler),
+    });
+}