Merge branch 'arith-dev' into 1444-failing-tests-for-testmxprandomadv…

…anced
Consensys · Dec 18, 2024 · 596b990 · 596b990
2 parents d9f9e3b + ffd428f
commit 596b990
Show file tree

Hide file tree

Showing 76 changed files with 1,113 additions and 449 deletions.
diff --git a/.github/workflows/reference-blockchain-tests.yml b/.github/workflows/reference-blockchain-tests.yml
@@ -43,7 +43,7 @@ jobs:
         run: cd ./linea-constraints; make zkevm_for_reference_tests.bin -B; cd ..
 
       - name: Generate block chain reference tests
-        run:  ./gradlew :reference-tests:generateBlockchainReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+        run: ./gradlew :reference-tests:generateBlockchainReferenceTests -Dorg.gradle.caching=true
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: fields,expand,expand,expand
@@ -63,10 +63,10 @@ jobs:
         run: mv ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/failedBlockchainReferenceTests.json ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/failedBlockchainReferenceTests-input.json
 
       - name: Run reference blockchain tests
-        run: ./gradlew referenceBlockchainTests -x spotlessCheck
-        timeout-minutes: 180
+        run: GOMEMLIMIT=32GiB ./gradlew referenceBlockchainTests -x spotlessCheck
+        timeout-minutes: 360
         env:
-          REFERENCE_TESTS_PARALLELISM: 7
+          REFERENCE_TESTS_PARALLELISM: 2
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
           GOCORSET_FLAGS: -wd --ansi-escapes=false --report --air

diff --git a/arithmetization/build.gradle b/arithmetization/build.gradle
@@ -14,8 +14,7 @@
  */
 
 plugins {
-  id 'java'
-  id 'java-library-distribution'
+  id 'java-library'
   id 'common-plugins'
   id 'com.github.hierynomus.license'
   id "de.undercouch.download"
@@ -35,16 +34,6 @@ apply from: rootProject.file("gradle/lint.gradle")
 apply from: rootProject.file("gradle/trace-files.gradle")
 
 dependencies {
-  /**
-   * Use pluginImplementation for dependencies that are specific to this plugin
-   * and are not already provided by Besu.
-   * These dependencies are the only ones that are included in the final jar.
-   */
-  pluginImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
-
-  // annotationProcessor generates the file META-INF/services/org.hyperledger.besu.plugin.BesuPlugin
-  annotationProcessor 'com.google.auto.service:auto-service'
-
   implementation "${besuArtifactGroup}:besu-datatypes"
   implementation "${besuArtifactGroup}:evm"
   implementation "${besuArtifactGroup}:plugin-api"
@@ -54,7 +43,9 @@ dependencies {
   implementation "${besuArtifactGroup}.internal:core"
   implementation "${besuArtifactGroup}.internal:rlp"
 
-  implementation 'com.google.auto.service:auto-service'
+  implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
+
+  implementation 'com.slack.api:slack-api-client'
 
   implementation 'info.picocli:picocli'
 
@@ -69,5 +60,18 @@ dependencies {
   testImplementation 'org.junit.platform:junit-platform-launcher'
 }
 
-apply from: rootProject.file("gradle/dist.gradle")
+jar {
+
+  version = calculateVersion()
+
+  manifest {
+    attributes(
+            'Specification-Title': archiveBaseName.get(),
+            'Specification-Version': calculateVersion(),
+            'Implementation-Title': archiveBaseName.get(),
+            'Implementation-Version': calculateVersion()
+    )
+  }
+}
+
 apply from: rootProject.file("gradle/publishing.gradle")
diff --git a/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/AccountSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/AccountSnapshot.java
@@ -245,6 +245,15 @@ public AccountSnapshot raiseNonceByOne() {
     return this;
   }
 
+  public AccountSnapshot setDeploymentNumber(Hub hub) {
+    return this.setDeploymentNumber(hub.transients.conflation().deploymentInfo());
+  }
+
+  public AccountSnapshot setDeploymentNumber(DeploymentInfo deploymentInfo) {
+    this.deploymentNumber(deploymentInfo.deploymentNumber(address));
+    return this;
+  }
+
   public AccountSnapshot setDeploymentInfo(Hub hub) {
     return this.setDeploymentInfo(hub.transients.conflation().deploymentInfo());
   }

diff --git a/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/ContextFragment.java b/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/ContextFragment.java
@@ -41,7 +41,7 @@
 @AllArgsConstructor
 public class ContextFragment implements TraceFragment {
   private final Hub hub;
-  private final Either<Integer, Integer> callFrameReference;
+  private final Either<Integer, Integer> callFrameReference; // left: id, right: context number
   private final MemoryRange returnDataRange;
   private final boolean updateReturnData;
 

diff --git a/...ization/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/StackFragment.java b/...ization/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/StackFragment.java
@@ -16,8 +16,10 @@
 package net.consensys.linea.zktracer.module.hub.fragment;
 
 import static com.google.common.base.Preconditions.*;
+import static com.google.common.primitives.Ints.min;
 import static net.consensys.linea.zktracer.module.hub.signals.TracedException.*;
 import static net.consensys.linea.zktracer.opcode.InstructionFamily.*;
+import static net.consensys.linea.zktracer.types.Utils.rightPadTo;
 
 import java.math.BigInteger;
 import java.util.List;
@@ -56,10 +58,12 @@ public final class StackFragment implements TraceFragment {
   private EWord hashInfoKeccak = EWord.ZERO;
   @Setter public Bytes hash;
   @Getter private final OpCode opCode;
+  @Getter private final int rawOpCode;
   @Setter private boolean jumpDestinationVettingRequired;
   @Setter private boolean validJumpDestination;
   private final State.TxState.Stamps stamps;
   private final CommonFragmentValues commonFragmentValues;
+  private final EWord pushValue;
 
   private StackFragment(
       final Hub hub,
@@ -74,6 +78,13 @@ private StackFragment(
     this.stackOps = stackOps;
     this.exceptions = exceptions;
     this.opCode = stack.getCurrentOpcodeData().mnemonic();
+    if (this.opCode != OpCode.INVALID) {
+      this.rawOpCode = 0xff & this.opCode.byteValue();
+    } else {
+      final int codeSize = hub.messageFrame().getCode().getBytes().size();
+      final int pc = hub.messageFrame().getPC();
+      this.rawOpCode = (pc < codeSize) ? 0xff & hub.messageFrame().getCode().getBytes().get(pc) : 0;
+    }
     this.hashInfoFlag =
         switch (this.opCode) {
               case SHA3 -> true;
@@ -109,7 +120,9 @@ private StackFragment(
           hub.currentFrame().frame().getStackItem(0).toUnsignedBigInteger();
       final BigInteger codeSize = BigInteger.valueOf(hub.currentFrame().code().getSize());
 
-      boolean prospectivePcNewIsInBounds = codeSize.compareTo(prospectivePcNew) > 0;
+      boolean noOutOfGasException = !Exceptions.outOfGasException(exceptions);
+      boolean prospectivePcNewIsInBounds =
+          codeSize.compareTo(prospectivePcNew) > 0 && noOutOfGasException;
 
       if (opCode.equals(OpCode.JUMPI)) {
         boolean nonzeroJumpCondition =
@@ -128,6 +141,24 @@ private StackFragment(
 
     this.stamps = hub.state().stamps();
     this.commonFragmentValues = commonFragmentValues;
+    this.pushValue = opCode.isPush() ? EWord.of(getPushValue(hub)) : EWord.ZERO;
+  }
+
+  private Bytes getPushValue(Hub hub) {
+    checkState(hub.opCode().isPush());
+
+    final int pc = hub.messageFrame().getPC();
+    if (pc + 1 >= hub.messageFrame().getCode().getSize()) {
+      return Bytes.EMPTY;
+    }
+
+    Bytes byteCode = hub.messageFrame().getCode().getBytes();
+    int nBytesToPush = (opCode.byteValue() & 0xff) - (OpCode.PUSH1.byteValue() & 0xff) + 1;
+    int nLeftoverBytes = byteCode.size() - (pc + 1);
+    Bytes partialPushValue = byteCode.slice(pc + 1, min(nLeftoverBytes, nBytesToPush));
+    return (nLeftoverBytes >= nBytesToPush)
+        ? partialPushValue
+        : rightPadTo(partialPushValue, nBytesToPush);
   }
 
   public static StackFragment prepare(
@@ -185,15 +216,11 @@ public Trace trace(Trace trace) {
             trace::pStackStackItemStamp3,
             trace::pStackStackItemStamp4);
 
-    EWord pushValue = EWord.ZERO;
     var it = stackOps.listIterator();
     while (it.hasNext()) {
       var i = it.nextIndex();
       var op = it.next();
-      final EWord eValue = EWord.of(op.value());
-      if (stack.getCurrentOpcodeData().isPush()) {
-        pushValue = eValue;
-      }
+      final EWord eValue = (i == 3 && opCode().isPush()) ? pushValue : EWord.of(op.value());
 
       heightTracers.get(i).apply(op.height());
       valHiTracers.get(i).apply(eValue.hi());
@@ -212,7 +239,7 @@ public Trace trace(Trace trace) {
         // Instruction details
         .pStackAlpha(UnsignedByte.of(stack.getCurrentOpcodeData().stackSettings().alpha()))
         .pStackDelta(UnsignedByte.of(stack.getCurrentOpcodeData().stackSettings().delta()))
-        .pStackInstruction(Bytes.of(stack.getCurrentOpcodeData().value()))
+        .pStackInstruction(Bytes.of(rawOpCode))
         .pStackStaticGas(staticGas)
         // Opcode families
         .pStackAccFlag(currentInstFamily == ACCOUNT)

diff --git a/...c/main/java/net/consensys/linea/zktracer/module/hub/fragment/account/AccountFragment.java b/...c/main/java/net/consensys/linea/zktracer/module/hub/fragment/account/AccountFragment.java
@@ -16,6 +16,7 @@
 package net.consensys.linea.zktracer.module.hub.fragment.account;
 
 import static com.google.common.base.Preconditions.*;
+import static net.consensys.linea.zktracer.module.hub.Trace.MULTIPLIER___DOM_SUB_STAMPS;
 import static net.consensys.linea.zktracer.types.AddressUtils.highPart;
 import static net.consensys.linea.zktracer.types.AddressUtils.isPrecompile;
 import static net.consensys.linea.zktracer.types.AddressUtils.lowPart;
@@ -51,8 +52,6 @@ public final class AccountFragment
   @Getter private final AccountSnapshot oldState;
   @Getter private final AccountSnapshot newState;
   @Setter private int deploymentNumberInfinity = 0; // retconned on conflation end
-  private final int deploymentNumber;
-  private final boolean isDeployment;
   @Setter private boolean existsInfinity = false; // retconned on conflation end
   @Setter private boolean requiresRomlex;
   private int codeFragmentIndex;
@@ -104,8 +103,6 @@ public AccountFragment(
 
     this.oldState = oldState;
     this.newState = newState;
-    deploymentNumber = newState.deploymentNumber();
-    isDeployment = newState.deploymentStatus();
     this.addressToTrim = addressToTrim;
     this.domSubStampsSubFragment = domSubStampsSubFragment;
 
@@ -177,7 +174,8 @@ public void resolvePostTransaction(
         new EphemeralAccount(oldState.address(), oldState.deploymentNumber());
     if (effectiveSelfDestructMap.containsKey(ephemeralAccount)) {
       final int selfDestructTime = effectiveSelfDestructMap.get(ephemeralAccount);
-      markedForSelfDestruct = hubStamp > selfDestructTime;
+      markedForSelfDestruct =
+          domSubStampsSubFragment.domStamp() > MULTIPLIER___DOM_SUB_STAMPS * selfDestructTime;
       markedForSelfDestructNew = hubStamp >= selfDestructTime;
     } else {
       markedForSelfDestruct = false;
@@ -187,13 +185,16 @@ public void resolvePostTransaction(
 
   @Override
   public void resolvePostConflation(Hub hub, WorldView world) {
-    deploymentNumberInfinity = hub.deploymentNumberOf(oldState.address());
-    existsInfinity = world.get(oldState.address()) != null;
+    deploymentNumberInfinity = hub.deploymentNumberOf(newState.address());
+    existsInfinity = world.get(newState.address()) != null;
     codeFragmentIndex =
         requiresRomlex
             ? hub.romLex()
                 .getCodeFragmentIndexByMetadata(
-                    ContractMetadata.make(oldState.address(), deploymentNumber, isDeployment))
+                    ContractMetadata.make(
+                        newState.address(),
+                        newState.deploymentNumber(),
+                        newState.deploymentStatus()))
             : 0;
   }
 }
diff --git a/...src/main/java/net/consensys/linea/zktracer/module/hub/fragment/common/CommonFragment.java b/...src/main/java/net/consensys/linea/zktracer/module/hub/fragment/common/CommonFragment.java
@@ -117,8 +117,14 @@ public Trace trace(Trace trace) {
         .gasCost(Bytes.ofUnsignedLong(commonFragmentValues.gasCostToTrace()))
         .gasNext(
             Bytes.ofUnsignedLong(isExec && isUnexceptional() ? commonFragmentValues.gasNext : 0))
-        .refundCounter(commonFragmentValues.gasRefund)
-        .refundCounterNew(commonFragmentValues.gasRefundNew)
+        .refundCounter(
+            (commonFragmentValues.hubProcessingPhase == TX_EXEC)
+                ? commonFragmentValues.gasRefund
+                : 0)
+        .refundCounterNew(
+            (commonFragmentValues.hubProcessingPhase == TX_EXEC)
+                ? commonFragmentValues.gasRefundNew
+                : 0)
         .twoLineInstruction(commonFragmentValues.TLI)
         .counterTli(twoLineInstructionCounter)
         .nonStackRows((short) commonFragmentValues.numberOfNonStackRows)

diff --git a/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/imc/ImcFragment.java b/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/imc/ImcFragment.java
@@ -137,7 +137,14 @@ public Trace trace(Trace trace) {
     trace.peekAtMiscellaneous(true);
 
     for (TraceSubFragment subFragment : moduleCalls) {
-      subFragment.trace(trace, hub.state.stamps());
+      if (subFragment instanceof MmuCall) {
+        MmuCall mmuCall = (MmuCall) subFragment;
+        if (mmuCall.traceMe()) {
+          subFragment.trace(trace, hub.state.stamps());
+        }
+      } else {
+        subFragment.trace(trace, hub.state.stamps());
+      }
     }
 
     if (childFrame != null) {

diff --git a/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/imc/mmu/MmuCall.java b/...ation/src/main/java/net/consensys/linea/zktracer/module/hub/fragment/imc/mmu/MmuCall.java
@@ -261,7 +261,7 @@ public static MmuCall invalidCodePrefix(final Hub hub) {
                     .frame()
                     .shadowReadMemory(0, hub.currentFrame().frame().memoryByteSize())))
         .sourceOffset(EWord.of(hub.messageFrame().getStackItem(0)))
-        .successBit(!Exceptions.invalidCodePrefix(currentExceptions));
+        .successBit(Exceptions.invalidCodePrefix(currentExceptions));
   }
 
   public static MmuCall revert(final Hub hub) {

diff --git a/...ain/java/net/consensys/linea/zktracer/module/hub/fragment/imc/mmu/opcode/ExtCodeCopy.java b/...ain/java/net/consensys/linea/zktracer/module/hub/fragment/imc/mmu/opcode/ExtCodeCopy.java
@@ -86,7 +86,8 @@ public long referenceSize() {
   @Override
   public void resolvePostConflation(Hub hub, WorldView world) {
     try {
-      sourceId(hub.romLex().getCodeFragmentIndexByMetadata(contract));
+      sourceId(
+          contract.underDeployment() ? 0 : hub.romLex().getCodeFragmentIndexByMetadata(contract));
     } catch (Exception ignored) {
       // Can be 0 in case the ext account is empty. In this case, no associated CFI
       sourceId(0);

diff --git a/...ava/net/consensys/linea/zktracer/module/hub/fragment/scenario/CreateScenarioFragment.java b/...ava/net/consensys/linea/zktracer/module/hub/fragment/scenario/CreateScenarioFragment.java
@@ -42,6 +42,16 @@ public boolean isAnyOf(CreateScenario... createScenarios) {
       }
       return false;
     }
+
+    public static CreateScenario switchToRevertingScenario(final CreateScenario currentScenario) {
+      return switch (currentScenario) {
+        case CREATE_FAILURE_CONDITION_WONT_REVERT -> CREATE_FAILURE_CONDITION_WILL_REVERT;
+        case CREATE_EMPTY_INIT_CODE_WONT_REVERT -> CREATE_EMPTY_INIT_CODE_WILL_REVERT;
+        case CREATE_NON_EMPTY_INIT_CODE_FAILURE_WONT_REVERT -> CREATE_NON_EMPTY_INIT_CODE_FAILURE_WILL_REVERT;
+        case CREATE_NON_EMPTY_INIT_CODE_SUCCESS_WONT_REVERT -> CREATE_NON_EMPTY_INIT_CODE_SUCCESS_WILL_REVERT;
+        default -> throw new IllegalArgumentException("unexpected Create scenario");
+      };
+    }
   }
 
   @Setter @Getter private CreateScenario scenario;