release 0.7.8, Backwards compatible trielog metadata api (#61)

* Revert "revert zkTrielogPlugin changes for older versions of shomei (#60)"
* add backward-compatible api for trielog construction

Signed-off-by: garyschulte <[email protected]>

Author: garyschulte

CHANGELOG.md

@@ -1,10 +1,5 @@
 # Changelog
 
-## unreleased
-
-## version 0.7.7
-* revert trielog metadata changes for trielogs until a new shomei release is tested [#60](https://github.com/Consensys/besu-shomei-plugin/pull/60) 
-
 ## version 0.7.6
 * update tracer to rc18, besu to 25.11.0-RC1-linea2 [#59](https://github.com/Consensys/besu-shomei-plugin/pull/59)
 * safe tracer for the besu-shomei-plugin  by matkt [#58](https://github.com/Consensys/besu-shomei-plugin/pull/58)

gradle.properties

@@ -1,3 +1,3 @@
-releaseVersion=0.7.7-SNAPSHOT
+releaseVersion=0.7.8-SNAPSHOT
 besuVersion=25.10.0-RC1-linea2
 arithmetizationVersion=beta-v4.0-rc18

src/main/java/net/consensys/shomei/rpc/BesuShomeiRpcPlugin.java

@@ -17,6 +17,7 @@
 import net.consensys.shomei.context.ShomeiContext;
 import net.consensys.shomei.context.ShomeiContext.ShomeiContextImpl;
 import net.consensys.shomei.rpc.methods.ShomeiGetTrieLog;
+import net.consensys.shomei.rpc.methods.ShomeiGetTrieLogMetadata;
 import net.consensys.shomei.rpc.methods.ShomeiGetTrieLogsByRange;
 
 import java.util.List;
@@ -36,7 +37,11 @@ public class BesuShomeiRpcPlugin implements BesuPlugin {
   @Override
   public void register(final ServiceManager serviceManager) {
     LOG.debug("Registering RPC plugins");
-    var methods = List.of(new ShomeiGetTrieLogsByRange(ctx), new ShomeiGetTrieLog(ctx));
+    var methods =
+        List.of(
+            new ShomeiGetTrieLogsByRange(ctx),
+            new ShomeiGetTrieLog(ctx),
+            new ShomeiGetTrieLogMetadata(ctx));
     serviceManager
         .getService(RpcEndpointService.class)
         .ifPresent(

src/main/java/net/consensys/shomei/rpc/methods/ShomeiGetTrieLogMetadata.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright ConsenSys 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package net.consensys.shomei.rpc.methods;
+
+import net.consensys.shomei.context.ShomeiContext;
+import net.consensys.shomei.trielog.PluginTrieLogLayer;
+
+import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest;
+
+public class ShomeiGetTrieLogMetadata implements PluginRpcMethod {
+  private final ShomeiContext ctx;
+
+  public ShomeiGetTrieLogMetadata(ShomeiContext ctx) {
+    this.ctx = ctx;
+  }
+
+  @Override
+  public String getNamespace() {
+    return "shomei";
+  }
+
+  @Override
+  public String getName() {
+    return "getTrieLogMetadata";
+  }
+
+  @Override
+  public Object execute(PluginRpcRequest rpcRequest) {
+    var params = rpcRequest.getParams();
+    Long blockNumber = Long.parseLong(params[0].toString());
+
+    return ctx.getTrieLogService()
+        .getTrieLogProvider()
+        .getTrieLogLayer(blockNumber)
+        .filter(t -> t instanceof PluginTrieLogLayer)
+        .map(t -> (PluginTrieLogLayer) t)
+        .orElse(null);
+  }
+}

src/main/java/net/consensys/shomei/trielog/PluginTrieLogLayer.java

@@ -14,11 +14,18 @@
  */
 package net.consensys.shomei.trielog;
 
+import net.consensys.shomei.cli.ShomeiCliOptions.ZkTraceComparisonFeature;
+
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.tuweni.bytes.Bytes;
@@ -37,18 +44,37 @@
  * trie changes as well.
  */
 @SuppressWarnings("unchecked")
+@JsonInclude(JsonInclude.Include.NON_ABSENT)
 public record PluginTrieLogLayer(
-    Hash blockHash,
-    Optional<Long> blockNumber,
-    Map<Address, TrieLog.LogTuple<AccountValue>> accounts,
-    Map<Address, TrieLog.LogTuple<Bytes>> code,
-    Map<Address, Map<StorageSlotKey, LogTuple<UInt256>>> storage,
-    boolean frozen)
+    @JsonIgnore Hash blockHash,
+    @JsonIgnore Optional<Long> blockNumber,
+    @JsonIgnore Map<Address, TrieLog.LogTuple<AccountValue>> accounts,
+    @JsonIgnore Map<Address, TrieLog.LogTuple<Bytes>> code,
+    @JsonIgnore Map<Address, Map<StorageSlotKey, LogTuple<UInt256>>> storage,
+    @JsonIgnore boolean frozen,
+    @JsonIgnore Optional<Integer> zkTraceComparisonFeature)
     implements TrieLog {
 
+  public PluginTrieLogLayer(
+      Hash blockHash,
+      Optional<Long> blockNumber,
+      Map<Address, TrieLog.LogTuple<AccountValue>> accounts,
+      Map<Address, TrieLog.LogTuple<Bytes>> code,
+      Map<Address, Map<StorageSlotKey, LogTuple<UInt256>>> storage,
+      boolean frozen) {
+    this(blockHash, blockNumber, accounts, code, storage, frozen, Optional.empty());
+  }
+
   /** Creates a new PluginTrieLogLayer with blockhash and empty maps to deserialize into. */
   public PluginTrieLogLayer(final Hash blockHash) {
-    this(blockHash, Optional.empty(), new HashMap<>(), new HashMap<>(), new HashMap<>(), true);
+    this(
+        blockHash,
+        Optional.empty(),
+        new HashMap<>(),
+        new HashMap<>(),
+        new HashMap<>(),
+        true,
+        Optional.empty());
   }
 
   /** Locks the layer so no new changes can be added; */
@@ -128,6 +154,45 @@ public Optional<AccountValue> getAccount(final Address address) {
     return Optional.ofNullable(accounts.get(address)).map(LogTuple::getUpdated);
   }
 
+  // JSON serialization methods for metadata
+  @JsonGetter("blockHash")
+  public String getBlockHashHex() {
+    return blockHash.toHexString();
+  }
+
+  @JsonGetter("blockNumber")
+  public Long getBlockNumberValue() {
+    return blockNumber.orElse(null);
+  }
+
+  @JsonGetter("zkTraceComparisonFeatures")
+  public List<String> getZkTraceComparisonFeatures() {
+    return zkTraceComparisonFeature
+        .map(ZkTraceComparisonFeature::fromMask)
+        .map(enumSet -> enumSet.stream().map(Enum::name).collect(Collectors.toList()))
+        .orElse(null);
+  }
+
+  @JsonGetter("zkTraceComparisonFeatureMask")
+  public Integer getZkTraceComparisonFeatureMask() {
+    return zkTraceComparisonFeature.orElse(null);
+  }
+
+  @JsonGetter("accountChangesCount")
+  public int getAccountChangesCount() {
+    return accounts.size();
+  }
+
+  @JsonGetter("codeChangesCount")
+  public int getCodeChangesCount() {
+    return code.size();
+  }
+
+  @JsonGetter("storageChangesCount")
+  public int getStorageChangesCount() {
+    return storage.values().stream().mapToInt(Map::size).sum();
+  }
+
   public String dump() {
     final StringBuilder sb = new StringBuilder();
     sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}');

src/main/java/net/consensys/shomei/trielog/ZkTrieLogFactory.java

@@ -109,7 +109,8 @@ public TrieLog create(final TrieLogAccumulator accumulator, final BlockHeader bl
           (Map<Address, LogTuple<AccountValue>>) accountsToUpdate,
           (Map<Address, LogTuple<Bytes>>) codeToUpdate,
           (Map<Address, Map<StorageSlotKey, LogTuple<UInt256>>>) storageToUpdate,
-          true);
+          true,
+          Optional.of(comparisonFeatureMask.get()));
     } finally {
       ctx.getBlockImportTraceProvider().clear();
     }
@@ -281,7 +282,6 @@ public static void writeTo(final TrieLog layer, final RLPOutput output) {
     output.writeBytes(layer.getBlockHash());
     // optionally write block number
     layer.getBlockNumber().ifPresent(output::writeLongScalar);
-
     for (final Address address : addresses) {
       output.startList(); // this change
       output.writeBytes(address);
@@ -327,6 +327,10 @@ public static void writeTo(final TrieLog layer, final RLPOutput output) {
       output.endList(); // this change
     }
 
+    // optionally write zkTraceComparisonFeature
+    if (layer instanceof PluginTrieLogLayer pluginLayer) {
+      pluginLayer.zkTraceComparisonFeature().ifPresent(output::writeInt);
+    }
     output.endList(); // container
   }
 
@@ -412,9 +416,17 @@ public static PluginTrieLogLayer readFrom(final RLPInput input) {
       // lenient leave list for forward compatible additions.
       input.leaveListLenient();
     }
+
+    // zkTraceComparisonFeature is optional (read as last element in container, before leaving)
+    Optional<Integer> zkTraceComparisonFeature =
+        Optional.of(!input.isEndOfCurrentList())
+            .filter(isPresent -> isPresent)
+            .map(__ -> input.readInt());
+
     input.leaveListLenient();
 
-    return new PluginTrieLogLayer(blockHash, blockNumber, accounts, code, storage, true);
+    return new PluginTrieLogLayer(
+        blockHash, blockNumber, accounts, code, storage, true, zkTraceComparisonFeature);
   }
 
   protected static <T> T nullOrValue(final RLPInput input, final Function<RLPInput, T> reader) {