Set minimum state pivot to be at least the block after

src/Nethermind/Nethermind.Synchronization.Test/IContainerSynchronizerTestExtensions.cs

@@ -15,7 +15,7 @@ public static class IContainerSynchronizerTestExtensions
     public static ContainerBuilder WithSuggestedHeaderOfStateRoot(this ContainerBuilder builder, Hash256 stateRoot)
     {
         IBlockTree blockTree = Substitute.For<IBlockTree>();
-        BlockHeader header = Build.A.BlockHeader.WithStateRoot(stateRoot).TestObject;
+        BlockHeader header = Build.A.BlockHeader.WithStateRoot(stateRoot).WithNumber(1).TestObject;
         blockTree.FindHeader(Arg.Any<long>()).Returns(header);
         blockTree.BestSuggestedHeader.Returns(header);
 

src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs

@@ -142,9 +142,9 @@ public void Will_deque_storage_request_if_high()
     [Test]
     public void Will_mark_progress_and_flush_when_finished()
     {
-        BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block
+        BlockTree blockTree = Build.A.BlockTree()
             .WithStateRoot(Keccak.EmptyTreeHash)
-            .TestObject).TestObject;
+            .OfChainLength(2).TestObject;
         TestMemDb memDb = new();
         SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = 1 };
         using ProgressTracker progressTracker = new(memDb, syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance);
@@ -163,7 +163,7 @@ public void Will_mark_progress_and_flush_when_finished()
 
     private ProgressTracker CreateProgressTracker(int accountRangePartition = 1)
     {
-        BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block.WithStateRoot(Keccak.EmptyTreeHash).TestObject).TestObject;
+        BlockTree blockTree = Build.A.BlockTree().WithStateRoot(Keccak.EmptyTreeHash).OfChainLength(2).TestObject;
         SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = accountRangePartition };
         return new(new MemDb(), syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance);
     }

src/Nethermind/Nethermind.Synchronization.Test/SnapSync/StateSyncPivotTest.cs

@@ -3,7 +3,6 @@
 
 using FluentAssertions;
 using Nethermind.Blockchain;
-using Nethermind.Blockchain.Synchronization;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Logging;
 using NSubstitute;
@@ -13,16 +12,18 @@ namespace Nethermind.Synchronization.Test.SnapSync;
 
 public class StateSyncPivotTest
 {
-    [TestCase(1000, 1000, 10, 100, 1022)]
-    [TestCase(900, 1000, 10, 50, 1022)]
-    [TestCase(900, 1000, 10, 100, 1022)]
-    [TestCase(900, 900, 32, 100, 900)]
+    [TestCase(1000, 1000, 10, 100, 1022, 0)]
+    [TestCase(900, 1000, 10, 50, 1022, 0)]
+    [TestCase(900, 1000, 10, 100, 1022, 0)]
+    [TestCase(900, 900, 32, 100, 900, 0)]
+    [TestCase(0, 300, 32, 100, 301, 300)]
     public void Will_set_new_best_header_some_distance_from_best_suggested(
         int originalBestSuggested,
         int newBestSuggested,
         int minDistance,
         int maxDistance,
-        int newBestHeader
+        int newPivotHeader,
+        int syncPivot
     )
     {
         IBlockTree blockTree = Substitute.For<IBlockTree>();
@@ -32,6 +33,8 @@ int newBestHeader
         Synchronization.FastSync.StateSyncPivot stateSyncPivot = new Synchronization.FastSync.StateSyncPivot(blockTree,
             new TestSyncConfig()
             {
+                PivotNumber = syncPivot.ToString(),
+                FastSync = true,
                 StateMinDistanceFromHead = minDistance,
                 StateMaxDistanceFromHead = maxDistance,
             }, LimboLogs.Instance);
@@ -40,6 +43,6 @@ int newBestHeader
         stateSyncPivot.GetPivotHeader().Should().NotBeNull();
 
         blockTree.BestSuggestedHeader.Returns(Build.A.BlockHeader.WithNumber(newBestSuggested).TestObject);
-        stateSyncPivot.GetPivotHeader().Number.Should().Be(newBestHeader);
+        stateSyncPivot.GetPivotHeader().Number.Should().Be(newPivotHeader);
     }
 }

src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncPivot.cs

@@ -68,7 +68,13 @@ public void UpdateHeaderForcefully()
         private void TrySetNewBestHeader(string msg)
         {
             BlockHeader bestSuggestedHeader = _blockTree.BestSuggestedHeader;
-            long targetBlockNumber = Math.Max(bestSuggestedHeader.Number + MultiSyncModeSelector.FastSyncLag - _syncConfig.StateMinDistanceFromHead, 0);
+            long targetBlockNumber = bestSuggestedHeader.Number + MultiSyncModeSelector.FastSyncLag - _syncConfig.StateMinDistanceFromHead;
+            targetBlockNumber = Math.Max(targetBlockNumber, 0);
+            // The new pivot must be at least one block after the sync pivot as the forward downloader does not
+            // download the block at the sync pivot which may cause state not found error if state was downloaded
+            // at exactly sync pivot.
+            targetBlockNumber = Math.Max(targetBlockNumber, _syncConfig.PivotNumberParsed + 1);
+
             BlockHeader bestHeader = _blockTree.FindHeader(targetBlockNumber);
             if (bestHeader is not null)
             {

src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs

@@ -25,6 +25,11 @@ private void ChangeState(SyncFeedState newState)
                 _taskCompletionSource ??= new TaskCompletionSource();
             }
 
+            if (CurrentState == SyncFeedState.Finished && newState == SyncFeedState.Finished)
+            {
+                return;
+            }
+
             if (CurrentState == SyncFeedState.Finished)
             {
                 throw new InvalidOperationException($"{GetType().Name} has already finished and cannot be {newState} again.");

src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs

@@ -137,6 +137,12 @@ public void UpdatePivot()
 
         public bool IsFinished(out SnapSyncBatch? nextBatch)
         {
+            if (!CanSync())
+            {
+                nextBatch = null;
+                return false;
+            }
+
             Interlocked.Increment(ref _reqCount);
 
             BlockHeader? pivotHeader = _pivot.GetPivotHeader();