Starting from transitioned initial anchor with tests

CHANGELOG.md

@@ -20,3 +20,5 @@ the [releases page](https://github.com/Consensys/teku/releases).
 - Added hidden option `--Xp2p-dumps-to-file-enabled` to enable saving p2p dumps to file.
 
 ### Bug Fixes
+
+- Fixed a checkpoint sync issue where Teku couldn't start when the finalized state has been transitioned with empty slot(s) 

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/TransitionedAnchorAcceptanceTest.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.google.common.io.Resources;
+import java.net.URL;
+import java.security.SecureRandom;
+import java.util.List;
+import java.util.stream.IntStream;
+import org.apache.tuweni.bytes.Bytes32;
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.api.schema.bellatrix.SignedBeaconBlockBellatrix;
+import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.teku.infrastructure.bytes.Bytes20;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
+import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator.InitialStateData;
+import tech.pegasys.teku.test.acceptance.dsl.TekuBeaconNode;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfigBuilder;
+import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeys;
+import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeystores;
+
+public class TransitionedAnchorAcceptanceTest extends AcceptanceTestBase {
+  private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
+  // SEED is chosen to have empty slots in the end of 3rd epoch.
+  // TODO: Good to find one with earlier end epoch empty slots
+  private static final byte[] SEED = new byte[] {0x11, 0x03, 0x04};
+
+  @Test
+  @SuppressWarnings("DoNotCreateSecureRandomDirectly")
+  void shouldMaintainValidatorsInMutableClient() throws Exception {
+    final SecureRandom rnd = SecureRandom.getInstance("SHA1PRNG");
+    rnd.setSeed(SEED);
+    final String networkName = "swift";
+
+    final List<BLSKeyPair> node1Validators =
+        IntStream.range(0, 16).mapToObj(__ -> BLSKeyPair.random(rnd)).toList();
+    final List<BLSKeyPair> node2Validators =
+        IntStream.range(0, 3).mapToObj(__ -> BLSKeyPair.random(rnd)).toList();
+
+    final ValidatorKeystores node1ValidatorKeystores =
+        new ValidatorKeystores(
+            node1Validators.stream()
+                .map(keyPair -> new ValidatorKeys(keyPair, Bytes32.random(rnd), false))
+                .toList());
+    // will be non-active in the beginning
+    final ValidatorKeystores node2ValidatorKeystores =
+        new ValidatorKeystores(
+            node2Validators.stream()
+                .map(keyPair -> new ValidatorKeys(keyPair, Bytes32.random(rnd), false))
+                .toList());
+
+    final InitialStateData genesis =
+        createGenesisGenerator()
+            .network(networkName)
+            .withAltairEpoch(UInt64.ZERO)
+            .withBellatrixEpoch(UInt64.ZERO)
+            .validatorKeys(node1ValidatorKeystores, node2ValidatorKeystores)
+            .generate();
+
+    final String node1FeeRecipient = "0xFE3B557E8Fb62b89F4916B721be55cEb828dBd73";
+    final TekuBeaconNode beaconNode1 =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withStubExecutionEngine()
+                .withJwtSecretFile(JWT_FILE)
+                .withNetwork(networkName)
+                .withRealNetwork()
+                .withInitialState(genesis)
+                .withAltairEpoch(UInt64.ZERO)
+                .withBellatrixEpoch(UInt64.ZERO)
+                .withValidatorProposerDefaultFeeRecipient(node1FeeRecipient)
+                .withReadOnlyKeystorePath(node1ValidatorKeystores)
+                .build());
+
+    beaconNode1.start();
+    beaconNode1.waitForFinalizationAfter(UInt64.valueOf(3));
+
+    final BeaconState finalizedState = beaconNode1.fetchFinalizedState();
+    final UInt64 finalizedSlot = finalizedState.getSlot();
+    final UInt64 finalizedEpoch = beaconNode1.getSpec().computeEpochAtSlot(finalizedSlot);
+    final UInt64 finalizedEpochFirstSlot =
+        beaconNode1.getSpec().computeStartSlotAtEpoch(finalizedEpoch);
+    assertNotEquals(finalizedSlot, finalizedEpochFirstSlot);
+    final UInt64 maxFinalizedSlot =
+        beaconNode1.getSpec().computeStartSlotAtEpoch(finalizedEpoch.plus(1));
+    System.out.println(
+        "State slot: "
+            + finalizedState.getSlot()
+            + ", maxSlot: "
+            + maxFinalizedSlot
+            + ", finalized Epoch: "
+            + finalizedEpoch);
+    assertTrue(finalizedState.getSlot().isLessThan(maxFinalizedSlot));
+    final BeaconState transitionedFinalizedState =
+        beaconNode1.getSpec().processSlots(finalizedState, maxFinalizedSlot);
+
+    final String node2FeeRecipient = "0xfe3b557E8Fb62B89F4916B721be55ceB828DBd55";
+    final TekuBeaconNode beaconNode2 =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withStubExecutionEngine()
+                .withJwtSecretFile(JWT_FILE)
+                .withNetwork(networkName)
+                .withRealNetwork()
+                .withInitialState(new InitialStateData(transitionedFinalizedState))
+                .withAltairEpoch(UInt64.ZERO)
+                .withBellatrixEpoch(UInt64.ZERO)
+                .withValidatorProposerDefaultFeeRecipient(node2FeeRecipient)
+                .withReadOnlyKeystorePath(node2ValidatorKeystores)
+                .withPeers(beaconNode1)
+                .build());
+
+    beaconNode2.start();
+    beaconNode2.waitUntilInSyncWith(beaconNode1);
+    beaconNode2.waitForBlockSatisfying(
+        block -> {
+          assertThat(block).isInstanceOf(SignedBeaconBlockBellatrix.class);
+          final SignedBeaconBlockBellatrix bellatrixBlock = (SignedBeaconBlockBellatrix) block;
+          assertEquals(
+              Bytes20.fromHexString(node2FeeRecipient),
+              bellatrixBlock.getMessage().getBody().executionPayload.feeRecipient);
+        });
+    beaconNode2.waitForNewFinalization();
+  }
+}

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java

@@ -456,6 +456,16 @@ public void waitForNewFinalization() {
         MINUTES);
   }
 
+  public void waitForFinalizationAfter(final UInt64 newFinalizedEpoch) {
+    LOG.debug("Wait for finalized block");
+    waitFor(
+        () ->
+            assertThat(fetchStateFinalityCheckpoints().orElseThrow().finalized.epoch)
+                .isGreaterThanOrEqualTo(newFinalizedEpoch),
+        9,
+        MINUTES);
+  }
+
   public void waitForMilestone(final SpecMilestone expectedMilestone) {
     waitForLogMessageContaining("Activating network upgrade: " + expectedMilestone.name());
   }
@@ -779,4 +789,13 @@ public void expectElOffline() throws IOException {
   public void expectElOnline() throws IOException {
     assertThat(fetchSyncStatus().data.elOffline).isFalse();
   }
+
+  public BeaconState fetchFinalizedState() throws IOException {
+    final Bytes beaconStateBytes =
+        httpClient.getAsBytes(
+            getRestApiUrl(),
+            "eth/v2/debug/beacon/states/finalized",
+            Map.of("Accept", "application/octet-stream"));
+    return spec.deserializeBeaconState(Bytes.wrap(beaconStateBytes));
+  }
 }

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/DepositProviderTest.java

@@ -482,6 +482,33 @@ void whenCallingForFinalizedSnapshotAndSnapshotAvailable_SnapshotReturned() {
         .isEqualTo(UInt64.valueOf(30));
   }
 
+  @Test
+  void whenUsingTransitionedAnchorPoint_FinalizationIsNotFailed() throws Exception {
+    setup(16);
+    Bytes32 finalizedBlockRoot = Bytes32.fromHexString("0x01");
+    updateStateEth1DepositIndex(10);
+    updateStateEth1DataDepositCount(10);
+    mockDepositsFromEth1Block(0, 20);
+    final AnchorPoint anchorPoint = mock(AnchorPoint.class);
+    final UpdatableStore store = mock(UpdatableStore.class);
+    when(recentChainData.getStore()).thenReturn(store);
+    when(store.getLatestFinalized()).thenReturn(anchorPoint);
+    final BeaconState transitionedState = spec.processSlots(state, state.getSlot().plus(2));
+    when(anchorPoint.getState()).thenAnswer(__ -> transitionedState);
+    when(eth1DataCache.getEth1DataAndHeight(eq(transitionedState.getEth1Data())))
+        .thenReturn(Optional.empty());
+
+    assertThat(depositProvider.getDepositMapSize()).isEqualTo(20);
+
+    depositProvider.onNewFinalizedCheckpoint(new Checkpoint(UInt64.ONE, finalizedBlockRoot), false);
+
+    assertThat(depositProvider.getDepositMapSize()).isEqualTo(10);
+    List<DepositWithIndex> availableDeposits = depositProvider.getAvailableDeposits();
+    assertThat(availableDeposits.size()).isEqualTo(10);
+
+    verify(eth1DataCache).getEth1DataAndHeight(eq(transitionedState.getEth1Data()));
+  }
+
   private void checkThatDepositProofIsValid(SszList<Deposit> deposits) {
     final SpecVersion genesisSpec = spec.getGenesisSpec();
     deposits.forEach(

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetStateByBlockRoot.java

@@ -69,6 +69,7 @@ public void handleRequest(RestApiRequest request) throws JsonProcessingException
     request.header(Header.CACHE_CONTROL, CACHE_NONE);
 
     final String blockId = request.getPathParameter(PARAMETER_BLOCK_ID);
+    // FIXME: not sure. Are we ok to return here transitioned state on keyword finalized?
     SafeFuture<Optional<BeaconState>> future = chainDataProvider.getBeaconStateByBlockId(blockId);
 
     request.respondAsync(

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -424,6 +424,10 @@ public UInt64 computeStartSlotAtEpoch(final UInt64 epoch) {
     return atEpoch(epoch).miscHelpers().computeStartSlotAtEpoch(epoch);
   }
 
+  public UInt64 computeEndSlotAtEpoch(final UInt64 epoch) {
+    return atEpoch(epoch).miscHelpers().computeEndSlotAtEpoch(epoch);
+  }
+
   public UInt64 computeEpochAtSlot(final UInt64 slot) {
     return atSlot(slot).miscHelpers().computeEpochAtSlot(slot);
   }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blocks/StateAndBlockSummary.java

@@ -33,11 +33,15 @@ public class StateAndBlockSummary implements BeaconBlockSummary {
   protected StateAndBlockSummary(final BeaconBlockSummary blockSummary, final BeaconState state) {
     checkNotNull(blockSummary);
     checkNotNull(state);
+    this.blockSummary = blockSummary;
+    this.state = state;
+    verifyStateAndBlockConsistency();
+  }
+
+  protected void verifyStateAndBlockConsistency() {
     checkArgument(
         blockSummary.getStateRoot().equals(state.hashTreeRoot()),
         "Block state root must match the supplied state");
-    this.blockSummary = blockSummary;
-    this.state = state;
   }
 
   public static StateAndBlockSummary create(final BeaconState state) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/AnchorPoint.java

@@ -38,22 +38,19 @@ public class AnchorPoint extends StateAndBlockSummary {
   private final Spec spec;
   private final Checkpoint checkpoint;
   private final boolean isGenesis;
+  private final boolean isTransitioned;
 
   private AnchorPoint(
       final Spec spec,
       final Checkpoint checkpoint,
       final BeaconState state,
       final BeaconBlockSummary blockSummary) {
     super(blockSummary, state);
-    checkArgument(
-        checkpoint.getRoot().equals(blockSummary.getRoot()), "Checkpoint and block must match");
-    checkArgument(
-        checkpoint.getEpochStartSlot(spec).isGreaterThanOrEqualTo(blockSummary.getSlot()),
-        "Block must be at or prior to the start of the checkpoint epoch");
-
     this.spec = spec;
     this.checkpoint = checkpoint;
     this.isGenesis = checkpoint.getEpoch().equals(SpecConfig.GENESIS_EPOCH);
+    this.isTransitioned = state.getSlot().isGreaterThan(blockSummary.getSlot());
+    verifyAnchor();
   }
 
   public static AnchorPoint create(
@@ -126,6 +123,42 @@ public static AnchorPoint fromInitialBlockAndState(
     return new AnchorPoint(spec, checkpoint, state, block);
   }
 
+  /**
+   * Skipping verification in the super class. All checks are made in {@link #verifyAnchor()}
+   * instead
+   */
+  @Override
+  protected void verifyStateAndBlockConsistency() {}
+
+  private void verifyAnchor() {
+    final UInt64 blockSlot = blockSummary.getSlot();
+    if (state.getSlot().isGreaterThan(blockSlot)) {
+      // the finalized state is transitioned with empty slot(s)
+      checkArgument(
+          blockSummary.getStateRoot().equals(state.getLatestBlockHeader().getStateRoot()),
+          "Block state root must match the latest block header state root in the state");
+      final int stateAndBlockRootsIndex =
+          blockSlot.mod(spec.getSlotsPerHistoricalRoot(blockSlot)).intValue();
+      checkArgument(
+          blockSummary
+              .getStateRoot()
+              .equals(state.getStateRoots().get(stateAndBlockRootsIndex).get()),
+          "Block state root must match the state root for the block slot %s in the state roots",
+          blockSlot);
+      checkArgument(
+          blockSummary.getRoot().equals(state.getBlockRoots().get(stateAndBlockRootsIndex).get()),
+          "Block root must match the root for the block slot %s in the block roots in the state",
+          blockSlot);
+    } else {
+      super.verifyStateAndBlockConsistency();
+    }
+    checkArgument(
+        checkpoint.getRoot().equals(blockSummary.getRoot()), "Checkpoint and block must match");
+    checkArgument(
+        checkpoint.getEpochStartSlot(spec).isGreaterThanOrEqualTo(blockSlot),
+        "Block must be at or prior to the start of the checkpoint epoch");
+  }
+
   public boolean isGenesis() {
     return isGenesis;
   }
@@ -146,6 +179,10 @@ public UInt64 getEpochStartSlot() {
     return checkpoint.getEpochStartSlot(spec);
   }
 
+  public boolean isTransitioned() {
+    return isTransitioned;
+  }
+
   @Override
   public boolean equals(final Object o) {
     if (this == o) {

ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/AnchorPointTest.java

@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.spec.datastructures.state;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
 
 import java.util.Optional;
@@ -21,11 +22,19 @@
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockAndState;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException;
+import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException;
 import tech.pegasys.teku.spec.util.DataStructureUtil;
+import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder;
+import tech.pegasys.teku.storage.storageSystem.StorageSystem;
 
 public class AnchorPointTest {
   private final Spec spec = TestSpecFactory.createDefault();
   private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec);
+  private final StorageSystem storageSystem =
+      InMemoryStorageSystemBuilder.create().specProvider(spec).build();
 
   @Test
   public void create_withCheckpointPriorToState() {
@@ -41,4 +50,30 @@ public void create_withCheckpointPriorToState() {
         .isInstanceOf(IllegalArgumentException.class)
         .hasMessageContaining("Block must be at or prior to the start of the checkpoint epoch");
   }
+
+  @Test
+  public void createFromInitialState_WhenFinalizedStateTransitionedWithAnEmptySlot()
+      throws SlotProcessingException, EpochProcessingException {
+    storageSystem.chainUpdater().initializeGenesis();
+
+    final UInt64 latestBlockHeaderEpoch = UInt64.valueOf(4);
+    final UInt64 latestBlockHeaderSlot = spec.computeEndSlotAtEpoch(latestBlockHeaderEpoch);
+    final SignedBlockAndState blockAndState =
+        storageSystem.chainUpdater().advanceChainUntil(latestBlockHeaderSlot);
+
+    // empty slot transition
+    final BeaconState postState =
+        spec.processSlots(blockAndState.getState(), latestBlockHeaderSlot.increment());
+
+    final AnchorPoint anchor = AnchorPoint.fromInitialState(spec, postState);
+
+    // verify finalized anchor
+    assertThat(anchor.getBlockSlot()).isEqualTo(latestBlockHeaderSlot);
+    assertThat(anchor.getStateRoot()).isEqualTo(blockAndState.getBlock().getStateRoot());
+    final Checkpoint expectedCheckpoint =
+        new Checkpoint(
+            latestBlockHeaderEpoch.plus(1),
+            blockAndState.getBlock().asHeader().getMessage().hashTreeRoot());
+    assertThat(anchor.getCheckpoint()).isEqualTo(expectedCheckpoint);
+  }
 }