Feature/Era1 import/export

src/Nethermind/Directory.Packages.props

@@ -12,6 +12,7 @@
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
     <PackageVersion Include="CommandLineParser" Version="2.9.1" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />
+    <PackageVersion Include="Cortex.SimpleSerialize" Version="0.2.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.0" />
     <PackageVersion Include="Crc32.NET" Version="1.2.0" />
     <PackageVersion Include="DnsClient" Version="1.7.0" />
@@ -65,14 +66,18 @@
     <PackageVersion Include="RocksDB" Version="8.3.2.39829" />
     <PackageVersion Include="SCrypt" Version="2.0.0.2" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
+    <PackageVersion Include="Snappier" Version="1.1.4" />
     <PackageVersion Include="Snappy.Standard" Version="0.2.0" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
+    <PackageVersion Include="System.Diagnostics.TextWriterTraceListener" Version="4.3.0" />
+    <PackageVersion Include="System.IO.Abstractions.TestingHelpers" Version="19.2.87" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="7.2.0" />
+    <PackageVersion Include="System.IO.Compression.ZipFile" Version="4.3.0" />
     <PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="20.0.4" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="20.0.4" />
     <PackageVersion Include="Websocket.Client" Version="5.0.0" />
   </ItemGroup>
-</Project>
+</Project>

src/Nethermind/Nethermind.Api/IBasicApi.cs

@@ -6,12 +6,14 @@
 using System.Linq;
 using Nethermind.Abi;
 using Nethermind.Api.Extensions;
+using Nethermind.Blockchain;
 using Nethermind.Config;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
 using Nethermind.Core.Timers;
 using Nethermind.Crypto;
 using Nethermind.Db;
+using Nethermind.Era1;
 using Nethermind.KeyStore;
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;
@@ -46,6 +48,7 @@ public interface IBasicApi
         ITimestamper Timestamper { get; }
         ITimerFactory TimerFactory { get; }
         IProcessExitSource? ProcessExit { get; set; }
+        IProcessExitToken? ProcessExitToken { get; set; }
 
         public IConsensusPlugin? GetConsensusPlugin() =>
             Plugins

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -228,6 +228,8 @@ public ISealEngine SealEngine
         public IList<IPublisher> Publishers { get; } = new List<IPublisher>(); // this should be called publishers
         public CompositePruningTrigger PruningTrigger { get; } = new();
         public IProcessExitSource? ProcessExit { get; set; }
+        public IProcessExitToken? ProcessExitToken { get; set; }
         public CompositeTxGossipPolicy TxGossipPolicy { get; } = new();
+        public IEraStore? EraStore { get; set; }
     }
 }

src/Nethermind/Nethermind.Blockchain/Era1/EraExporter.cs

@@ -0,0 +1,182 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.IO.Abstractions;
+using Nethermind.Blockchain.Receipts;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
+using Nethermind.Core.Specs;
+using System;
+using System.Threading;
+using System.IO;
+using System.Threading.Tasks;
+using Nethermind.Era1;
+using System.Linq;
+using Nethermind.Blockchain.Era1;
+
+namespace Nethermind.Blockchain;
+public class EraExporter : IEraExporter
+{
+    private const int MergeBlock = 15537393;
+    private readonly IFileSystem _fileSystem;
+    private readonly IBlockTree _blockTree;
+    private readonly IReceiptStorage _receiptStorage;
+    private readonly ISpecProvider _specProvider;
+    private readonly string _networkName;
+
+    public event EventHandler<ExportProgressArgs> ExportProgress;
+    public event EventHandler<VerificationProgressArgs> VerificationProgress;
+
+    public string NetworkName => _networkName;
+
+    public EraExporter(
+        IFileSystem fileSystem,
+        IBlockTree blockTree,
+        IReceiptStorage receiptStorage,
+        ISpecProvider specProvider,
+        string networkName)
+    {
+        _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+        _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree));
+        _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage));
+        _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider));
+        if (string.IsNullOrWhiteSpace(networkName)) throw new ArgumentException("Cannot be null or whitespace.", nameof(specProvider));
+        _networkName = networkName.Trim().ToLower();
+    }
+
+    public async Task Export(
+        string destinationPath,
+        long start,
+        long end,
+        int size = EraWriter.MaxEra1Size,
+        CancellationToken cancellation = default)
+    {
+        if (destinationPath is null) throw new ArgumentNullException(nameof(destinationPath));
+        if (_fileSystem.File.Exists(destinationPath)) throw new ArgumentException(nameof(destinationPath), $"Cannot be a file.");
+        if (size < 1) throw new ArgumentOutOfRangeException(nameof(size), size, $"Must be greater than 0.");
+        if (size > EraWriter.MaxEra1Size) throw new ArgumentOutOfRangeException(nameof(size), size, $"Cannot be greater than {EraWriter.MaxEra1Size}.");
+        if (!_fileSystem.Directory.Exists(destinationPath))
+        {
+            //TODO look into permission settings - should it be set?
+            _fileSystem.Directory.CreateDirectory(destinationPath);
+        }
+
+        DateTime startTime = DateTime.Now;
+        DateTime lastProgress = DateTime.Now;
+
+        int totalProcessed = 0;
+        int processedSinceLast = 0;
+        int txProcessedSinceLast = 0;
+
+        for (long i = start; i <= end; i += size)
+        {
+            string filePath = Path.Combine(
+               destinationPath,
+               EraWriter.Filename(_networkName, i / size, Keccak.Zero));
+            using EraWriter? builder = EraWriter.Create(_fileSystem.File.Create(filePath), _specProvider);
+
+            //TODO read directly from RocksDb with range reads
+            for (var y = i; y <= end; y++)
+            {
+                Block? block = _blockTree.FindBlock(y, BlockTreeLookupOptions.DoNotCreateLevelIfMissing);
+
+                if (block == null)
+                {
+                    throw new EraException($"Could not find a block with number {y}.");
+                }
+
+                TxReceipt[]? receipts = _receiptStorage.Get(block);
+                if (receipts == null)
+                {
+                    //Can this even happen?
+                    throw new EraException($"Could not find receipts for block {block.ToString(Block.Format.FullHashAndNumber)}");
+                }
+                if (!await builder.Add(block, receipts, cancellation) || y == end)
+                {
+                    byte[] root = await builder.Finalize();
+                    builder.Dispose();
+                    string rename = Path.Combine(
+                                            destinationPath,
+                                            EraWriter.Filename(_networkName, i / size, new Hash256(root)));
+                    _fileSystem.File.Move(
+                        filePath,
+                        rename, true);
+                    break;
+                }
+                totalProcessed++;
+                txProcessedSinceLast += block.Transactions.Length;
+                processedSinceLast++;
+                TimeSpan elapsed = DateTime.Now.Subtract(lastProgress);
+                if (elapsed.TotalSeconds > TimeSpan.FromSeconds(10).TotalSeconds)
+                {
+                    ExportProgress?.Invoke(this, new ExportProgressArgs(
+                        end - start,
+                        totalProcessed,
+                        processedSinceLast,
+                        txProcessedSinceLast,
+                        elapsed,
+                        DateTime.Now.Subtract(startTime)));
+                    lastProgress = DateTime.Now;
+                    processedSinceLast = 0;
+                    txProcessedSinceLast = 0;
+                }
+            }
+        }
+    }
+    /// <summary>
+    /// Verifies all era1 archives from a directory, with an expected accumulator list from a hex encoded file.
+    /// </summary>
+    /// <param name="eraDirectory"></param>
+    /// <param name="accumulatorFile"></param>
+    /// <param name="cancellation"></param>
+    /// <exception cref="EraVerificationException">If the verification fails.</exception>
+    public async Task VerifyEraFiles(string eraDirectory, string accumulatorFile, CancellationToken cancellation = default)
+    {
+        string[] eraFiles = EraReader.GetAllEraFiles(eraDirectory, _networkName).ToArray();
+        string[] lines = await _fileSystem.File.ReadAllLinesAsync(accumulatorFile, cancellation);
+
+        byte[][] accumulators = lines.Select(s => Bytes.FromHexString(s)).ToArray();
+
+        await VerifyEraFiles(eraFiles, accumulators);
+    }
+    /// <summary>
+    /// Verifies all era1 files, with an expected accumulator list.
+    /// </summary>
+    /// <param name="eraFiles"></param>
+    /// <param name="expectedAccumulators"></param>
+    /// <param name="cancellation"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentNullException"></exception>
+    /// <exception cref="ArgumentException"></exception>
+    /// <exception cref="EraVerificationException">If the verification fails.</exception>
+    public async Task VerifyEraFiles(string[] eraFiles, byte[][] expectedAccumulators, CancellationToken cancellation = default)
+    {
+        if (expectedAccumulators is null) throw new ArgumentNullException(nameof(expectedAccumulators));
+        if (eraFiles is null) throw new ArgumentNullException(nameof(eraFiles));
+        if (eraFiles.Length == 0)
+            throw new ArgumentException("Must have at least one file.", nameof(eraFiles));
+        if (eraFiles.Length != expectedAccumulators.Length)
+            throw new ArgumentException("Must have an equal amount of files and accumulators.", nameof(eraFiles));
+        if (expectedAccumulators.Any(a=>a.Length != 32))
+            throw new ArgumentException("All accumulators must have a length of 32 bytes.", nameof(eraFiles));
+
+        DateTime startTime = DateTime.Now; 
+        DateTime lastProgress = DateTime.Now;
+        for (int i = 0; i < eraFiles.Length; i++)
+        {
+            using EraReader reader = await EraReader.Create(_fileSystem.File.OpenRead(eraFiles[i]), cancellation);
+            if (!await reader.VerifyAccumulator(expectedAccumulators[i], _specProvider, cancellation))
+            {
+                throw new EraVerificationException($"The accumulator for the archive '{eraFiles[i]}' does not match the expected accumulator '{expectedAccumulators[i].ToHexString()}'.");
+            }
+
+            TimeSpan elapsed = DateTime.Now.Subtract(lastProgress);
+            if (elapsed.TotalSeconds > TimeSpan.FromSeconds(10).TotalSeconds)
+            {
+                VerificationProgress?.Invoke(this, new VerificationProgressArgs(i, eraFiles.Length, DateTime.Now.Subtract(startTime)));
+                lastProgress = DateTime.Now;
+            }
+        }
+    }
+}

src/Nethermind/Nethermind.Blockchain/Era1/EraStore.cs

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Abstractions;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Nethermind.Core;
+using Nethermind.Era1;
+
+namespace Nethermind.Blockchain;
+public class EraStore: IEraStore
+{
+    private readonly char[] _eraSeparator = ['-'];
+    private readonly Dictionary<long, string> _epochs;
+    private readonly IFileSystem _fileSystem;
+
+    public int EpochCount => _epochs.Count;
+    public int BiggestEpoch { get; private set; }
+    public int SmallestEpoch { get; private set; }
+
+    public EraStore(string[] eraFiles) : this(eraFiles, new FileSystem()) { }
+    public EraStore(string[] eraFiles, IFileSystem fileSystem)
+    {
+        _epochs = new();
+        foreach (var file in eraFiles)
+        {
+            string[] parts = Path.GetFileName(file).Split(_eraSeparator);
+            int epoch;
+            if (parts.Length != 3 || !int.TryParse(parts[1], out epoch) || epoch < 0)
+                throw new ArgumentException($"Malformed Era1 file '{file}'.",nameof(eraFiles));
+            _epochs[epoch] = file;
+            if (epoch > BiggestEpoch)
+                BiggestEpoch = epoch;
+            if (epoch < SmallestEpoch)
+                SmallestEpoch = epoch;
+        }
+        _fileSystem = fileSystem;
+    }
+    public bool HasEpoch(long epoch) => _epochs.ContainsKey(epoch);
+
+    public Task<EraReader> GetReader(long epoch, CancellationToken cancellation = default)
+    {
+        GuardMissingEpoch(epoch);
+        return EraReader.Create(_epochs[epoch], _fileSystem, cancellation);
+    }
+
+    public string GetReaderPath(long epoch)
+    {
+        GuardMissingEpoch(epoch);
+        return _epochs[epoch];
+    }
+
+    public async Task<Block?> FindBlock(long number, CancellationToken cancellation = default)
+    {
+        ThrowIfNegative(number);
+
+        long partOfEpoch = number == 0 ? 0 : number / EraWriter.MaxEra1Size;
+        if (!_epochs.ContainsKey(partOfEpoch))
+            return null;
+        using (EraReader reader = await EraReader.Create(_fileSystem.File.OpenRead(_epochs[partOfEpoch]), cancellation))
+        {
+            (Block b, _, _) = await reader.GetBlockByNumber(number, cancellation);
+            return b;
+        }
+    }
+    public async Task<(Block?, TxReceipt[]?)> FindBlockAndReceipts(long number, CancellationToken cancellation = default)
+    {
+        ThrowIfNegative(number);
+
+        long partOfEpoch = number == 0 ? 0 : number / EraWriter.MaxEra1Size;
+        if (!_epochs.ContainsKey(partOfEpoch))
+            return (null, null);
+        using (EraReader reader = await EraReader.Create(_fileSystem.File.OpenRead(_epochs[partOfEpoch]), cancellation))
+        {
+            (Block b, TxReceipt[] r, _) = await reader.GetBlockByNumber(number, cancellation);
+            return (b, r);
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static void ThrowIfNegative(long number)
+    {
+        if (number < 0)
+            throw new ArgumentOutOfRangeException(nameof(number), number, "Cannot be negative.");
+    }
+
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private void GuardMissingEpoch(long epoch)
+    {
+        if (!HasEpoch(epoch))
+            throw new ArgumentOutOfRangeException($"Does not contain epoch.", epoch, nameof(epoch));
+    }
+}

src/Nethermind/Nethermind.Blockchain/Era1/EraVerificationException.cs

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Runtime.Serialization;
+
+namespace Nethermind.Blockchain;
+public class EraVerificationException : Exception
+{
+    public EraVerificationException()
+    {
+    }
+
+    public EraVerificationException(string message) : base(message)
+    {
+    }
+
+    public EraVerificationException(string message, Exception innerException) : base(message, innerException)
+    {
+    }
+}