Adding basic version of DUMP and RESTORE commands

libs/common/Crc64.cs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c
+/// </summary>
+public static class Crc64
+{
+    /// <summary>
+    /// Polynomial (same as redis)
+    /// </summary>
+    private const ulong POLY = 0xad93d23594c935a9UL;
+
+    /// <summary>
+    /// Reverse all bits in a 64-bit value (bit reflection).
+    /// Only used for data_len == 64 in this code.
+    /// </summary>
+    private static ulong Reflect64(ulong data)
+    {
+        // swap odd/even bits
+        data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1);
+        // swap consecutive pairs
+        data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2);
+        // swap nibbles
+        data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4);
+        // swap bytes, then 2-byte pairs, then 4-byte pairs
+        data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data);
+        return data;
+    }
+
+    /// <summary>
+    /// A direct bit-by-bit CRC64 calculation (like _crc64 in C).
+    /// </summary>
+    private static ulong Crc64Bitwise(ReadOnlySpan<byte> data)
+    {
+        ulong crc = 0;
+
+        foreach (var c in data)
+        {
+            for (byte i = 1; i != 0; i <<= 1)
+            {
+                // interpret the top bit of 'crc' and current bit of 'c'
+                var bitSet = (crc & 0x8000000000000000UL) != 0;
+                var cbit = (c & i) != 0;
+
+                // if cbit flips the sense, invert bitSet
+                if (cbit)
+                    bitSet = !bitSet;
+
+                // shift
+                crc <<= 1;
+
+                // apply polynomial if needed
+                if (bitSet)
+                    crc ^= POLY;
+            }
+
+            // ensure it stays in 64 bits
+            crc &= 0xffffffffffffffffUL;
+        }
+
+        // reflect and XOR, per standard
+        crc &= 0xffffffffffffffffUL;
+        crc = Reflect64(crc) ^ 0x0000000000000000UL;
+        return crc;
+    }
+
+    /// <summary>
+    /// Computes crc64
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    public static byte[] Hash(ReadOnlySpan<byte> data)
+    {
+        var bitwiseCrc = Crc64Bitwise(data);
+        return BitConverter.GetBytes(bitwiseCrc);
+    }
+}

libs/common/RedisLengthEncodingUtils.cs

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Linq;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Utils for working with redis length encoding
+/// </summary>
+public static class RedisLengthEncodingUtils
+{
+    /// <summary>
+    /// Decodes the redis length encoded length and returns payload start
+    /// </summary>
+    /// <param name="buff"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentException"></exception>
+    public static (long length, byte payloadStart) DecodeLength(ref ReadOnlySpan<byte> buff)
+    {
+        // remove the value type byte
+        var encoded = buff.Slice(1);
+
+        if (encoded.Length == 0)
+            throw new ArgumentException("Encoded length cannot be empty.", nameof(encoded));
+
+        var firstByte = encoded[0];
+        return (firstByte >> 6) switch
+        {
+            // 6-bit encoding
+            0 => (firstByte & 0x3F, 1),
+            // 14-bit encoding
+            1 when encoded.Length < 2 => throw new ArgumentException("Not enough bytes for 14-bit encoding."),
+            1 => (((firstByte & 0x3F) << 8) | encoded[1], 2),
+            // 32-bit encoding
+            2 when encoded.Length < 5 => throw new ArgumentException("Not enough bytes for 32-bit encoding."),
+            2 => ((long)((encoded[1] << 24) | (encoded[2] << 16) | (encoded[3] << 8) | encoded[4]), 5),
+            _ => throw new ArgumentException("Invalid encoding type.", nameof(encoded))
+        };
+    }
+
+    /// <summary>
+    /// Encoded payload length to redis encoded payload length
+    /// </summary>
+    /// <param name="length"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentOutOfRangeException"></exception>
+    public static byte[] EncodeLength(long length)
+    {
+        switch (length)
+        {
+            // 6-bit encoding (length ≤ 63)
+            case < 1 << 6:
+                return [(byte)(length & 0x3F)]; // 00xxxxxx
+            // 14-bit encoding (64 ≤ length ≤ 16,383)
+            case < 1 << 14:
+                {
+                    var firstByte = (byte)(((length >> 8) & 0x3F) | (1 << 6)); // 01xxxxxx
+                    var secondByte = (byte)(length & 0xFF);
+                    return [firstByte, secondByte];
+                }
+            // 32-bit encoding (length ≤ 4,294,967,295)
+            case <= 0xFFFFFFFF:
+                {
+                    var firstByte = (byte)(2 << 6); // 10xxxxxx
+                    var lengthBytes = BitConverter.GetBytes((uint)length); // Ensure unsigned
+                    if (BitConverter.IsLittleEndian)
+                    {
+                        Array.Reverse(lengthBytes); // Convert to big-endian
+                    }
+                    return new[] { firstByte }.Concat(lengthBytes).ToArray();
+                }
+            default:
+                throw new ArgumentOutOfRangeException("Length exceeds maximum allowed for Redis encoding (4,294,967,295).");
+        }
+    }
+}

libs/host/Configuration/Options.cs

@@ -499,6 +499,10 @@ internal sealed class Options
         [OptionValidation]
         [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")]
         public bool? FailOnRecoveryError { get; set; }
+
+        [OptionValidation]
+        [Option("skip-checksum-validation", Default = false, Required = false, HelpText = "Skip checksum validation")]
+        public bool? SkipChecksumValidation { get; set; }
 
         /// <summary>
         /// This property contains all arguments that were not parsed by the command line argument parser
@@ -708,7 +712,8 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexResizeFrequencySecs = IndexResizeFrequencySecs,
                 IndexResizeThreshold = IndexResizeThreshold,
                 LoadModuleCS = LoadModuleCS,
-                FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault()
+                FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(),
+                SkipChecksumValidation = SkipChecksumValidation.GetValueOrDefault(),
             };
         }
 

libs/resources/RespCommandsDocs.json

@@ -2586,6 +2586,50 @@
       }
     ]
   },
+  {
+    "Command": "DUMP",
+    "Name": "DUMP",
+    "Summary": "Returns a serialized representation of the value stored at a key.",
+    "Group": "Generic",
+    "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      }
+    ]
+  },
+  {
+    "Command": "RESTORE",
+    "Name": "RESTORE",
+    "Summary": "Creates a key from the serialized representation of a value.",
+    "Group": "Generic",
+    "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "TTL",
+        "DisplayText": "ttl",
+        "Type": "Integer"
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "SERIALIZEDVALUE",
+        "DisplayText": "serialized-value",
+        "Type": "String"
+      }
+    ]
+  },
   {
     "Command": "GET",
     "Name": "GET",

libs/resources/RespCommandsInfo.json

@@ -1395,6 +1395,56 @@
       }
     ]
   },
+  {
+    "Command": "RESTORE",
+    "Name": "RESTORE",
+    "Arity": -4,
+    "Flags": "DenyOom, Write",
+    "FirstKey": 1,
+    "LastKey": 1,
+    "Step": 1,
+    "AclCategories": "KeySpace, Dangerous",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 0,
+          "Limit": 0
+        },
+        "Flags": "OW, Update"
+      }
+    ]
+  },
+  {
+    "Command": "DUMP",
+    "Name": "DUMP",
+    "Arity": 2,
+    "Flags": "ReadOnly",
+    "FirstKey": 1,
+    "LastKey": 1,
+    "Step": 1,
+    "AclCategories": "KeySpace",
+    "KeySpecifications": [
+      {
+        "BeginSearch": {
+          "TypeDiscriminator": "BeginSearchIndex",
+          "Index": 1
+        },
+        "FindKeys": {
+          "TypeDiscriminator": "FindKeysRange",
+          "LastKey": 0,
+          "KeyStep": 1,
+          "Limit": 0
+        },
+        "Flags": "RO, Access"
+      }
+    ]
+  },
   {
     "Command": "GET",
     "Name": "GET",

libs/server/API/GarnetApi.cs

@@ -98,9 +98,9 @@ public GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMe
         /// <inheritdoc />
         public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output)
             => storageSession.TTL(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true);
-
+        
         #endregion
-
+        
         #region EXPIRETIME
 
         /// <inheritdoc />

libs/server/API/GarnetWatchApi.cs

@@ -74,7 +74,7 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM
             garnetApi.WATCH(new ArgSlice(ref key), storeType);
             return garnetApi.PTTL(ref key, storeType, ref output);
         }
-
+        
         #endregion
 
         #region EXPIRETIME

libs/server/Resp/BasicCommands.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 using Garnet.common;

libs/server/Resp/CmdStrings.cs

@@ -215,6 +215,7 @@ static partial class CmdStrings
         public static ReadOnlySpan<byte> RESP_ERR_INCR_SUPPORTS_ONLY_SINGLE_PAIR => "ERR INCR option supports a single increment-element pair"u8;
         public static ReadOnlySpan<byte> RESP_ERR_INVALID_BITFIELD_TYPE => "ERR Invalid bitfield type. Use something like i16 u8. Note that u64 is not supported but i64 is"u8;
         public static ReadOnlySpan<byte> RESP_ERR_SCRIPT_FLUSH_OPTIONS => "ERR SCRIPT FLUSH only support SYNC|ASYNC option"u8;
+        public static ReadOnlySpan<byte> RESP_ERR_KEY_ALREADY_EXISTS => "ERR Key already exists"u8;
 
         /// <summary>
         /// Response string templates