Merge branch 'master' into keepaliveandtimeoutmechanism

DiceDB · Nov 14, 2024 · 1847b8e · 1847b8e
2 parents 4d1a919 + b127d96
commit 1847b8e
Show file tree

Hide file tree

Showing 403 changed files with 35,282 additions and 18,424 deletions.
diff --git a/.air.toml b/.air.toml
@@ -10,7 +10,7 @@ tmp_dir = "tmp"
   exclude_regex = []
   exclude_unchanged = false
   follow_symlink = false
-  full_bin = "GO_ENV=development ./dicedb --enable-multithreading=false --enable-profiling=false"
+  full_bin = "GO_ENV=development ./dicedb --enable-multithreading=true --enable-profiling=false"
   include_dir = []
   include_ext = ["go", "tpl", "tmpl", "yaml", "env"]
   kill_delay = "1s"

diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml
@@ -0,0 +1,21 @@
+name: Weekly Trivy Scan
+
+on:
+  schedule:
+    - cron: '0 0 * * 1' # Run every Monday at midnight
+  workflow_dispatch:
+
+jobs:
+  trivy_scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.56.2
+
+      - name: Run Trivy scan
+        run: |
+          trivy fs --security-checks vuln .
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -17,6 +17,9 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v4
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
@@ -29,19 +32,21 @@ jobs:
         with:
           images: dicedb/dicedb
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image (with cache)
         id: push
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-
+          cache-from: type=gha # This tells Buildx to attempt to use the GitHub Actions cache
+          cache-to: type=gha,mode=max # This stores the generated cache in the GitHub Actions cache
+
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v1
         with:
           subject-name: docker.io/dicedb/dicedb
           subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: true
diff --git a/.github/workflows/full-test-suite.yml b/.github/workflows/full-test-suite.yml
@@ -33,11 +33,19 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with: # Cache the Go modules
+          path: |
+            vendor/
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
       - name: Install dependencies
-        run: go get .
+        run: go get -v . # Check if there is any version changes from the cache
       - name: Build
         run: make build
       - name: Run Unit tests
         run: make unittest
       - name: Run Integration tests
-        run: make test
+        run: make test
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -1,9 +1,15 @@
 name: linter
+
 on:
   push:
-    branches: [ master ]
+    branches: [master]
+    paths:
+      - '**/*.go'
+
   pull_request:
-    branches: [ master ]
+    branches: [master]
+    paths:
+      - '**/*.go'
 
 permissions:
   contents: read

diff --git a/CONTRIBUTING/README.md b/CONTRIBUTING/README.md
@@ -101,6 +101,7 @@ $ git add <updated files>
 $ git commit -m "Meaningful message for the update"
 # pull new changes
 $ git checkout master
+$ git fetch upstream
 $ git merge upstream/master
 $ git checkout <your issue branch>
 $ git rebase master

diff --git a/README.md b/README.md
@@ -2,14 +2,15 @@ DiceDB
 ===
 
 <a href="https://dicedb.io">![slatedb.io](https://img.shields.io/badge/site-dicedb.io-00A1FF?style=flat-square)</a>
-<a href="https://discord.gg/6r8uXWtXh7">![Discord](https://img.shields.io/discord/1232385660460204122?style=flat-square)</a>
 <a href="https://dicedb.io/get-started/installation/">![Docs](https://img.shields.io/badge/docs-00A1FF?style=flat-square)</a>
+<a target="_blank" href="https://discord.gg/6r8uXWtXh7"><img src="https://dcbadge.limes.pink/api/server/6r8uXWtXh7?style=flat" alt="discord community" /></a>
 
-DiceDB is a redis-compliant, in-memory, real-time, and reactive database optimized for modern hardware and for building and scaling truly real-time applications. 
+DiceDB is a redis-compliant, reactive, scalable, highly available, unified cache optimized for modern hardware.
 
 We are looking for Early Design Partners, so, if you want to evaluate DiceDB, [block our calendar](https://cal.com/dicedb-arpit). always up for a chat.
 
-> Note: DiceDB is still in development and it supports a subset of Redis commands. So, please do not use it in production. But, feel free to go through the [open issues](https://github.com/DiceDB/dice/issues) and contribute to help us speed up the development.
+> [!CAUTION]
+> DiceDB is under development and it supports a subset of Redis commands. So, please do not use it in production. But, feel free to go through the [open issues](https://github.com/DiceDB/dice/issues) and contribute to help us speed up the development.
 
 ## Want to contribute?
 
@@ -42,18 +43,11 @@ docker run -p 7379:7379 dicedb/dicedb --enable-multithreading --enable-watch
 The above command will start the DiceDB server running locally on the port `7379` and you can connect
 to it using [DiceDB CLI](https://github.com/DiceDB/dicedb-cli) and SDKs.
 
-### Setting up CLI
-
-The best way to connect to DiceDB is using [DiceDB CLI](https://github.com/DiceDB/dicedb-cli) and you can install it by running the following command
-
-```bash
-sudo su
-curl -sL https://raw.githubusercontent.com/DiceDB/dicedb-cli/refs/heads/master/install.sh | sh
-```
+> [!TIP]
+> Since DiceDB is a drop-in replacement for Redis, you can also use any Redis CLI and SDK to connect to DiceDB.
 
-If you are working on unsupported OS (as per above script), you can always follow the installation instructions mentioned in the [dicedb/cli](https://github.com/DiceDB/dicedb-cli) repository.
 
-## For development and contributions
+### Setting up DiceDB from source for development and contributions
 
 To run DiceDB for local development or running from source, you will need
 
@@ -120,16 +114,9 @@ go run main.go -init-config
 
 ```
 
-#### For Windows Users:
-If you're using Windows, it is recommended to use Windows Subsystem for Linux (WSL) or WSL 2 to run the above commands seamlessly in a Linux-like environment.
-
-Alternatively, you can:
-
-Create a directory at `C:\ProgramData\dice` and run the following command to generate the configuration file:
-```bash
-go run main.go -init-config
-```
-For a smoother experience, we highly recommend using WSL.
+> [!NOTE]
+> #### For Windows Users:
+> If you're using Windows, it is recommended to use Windows Subsystem for Linux (WSL) or WSL 2 to run the above commands seamlessly in a Linux-like environment.
 
 #### Additional Configuration Options:
 
@@ -144,18 +131,25 @@ If you'd like to output the configuration file to a specific location, you can s
 go run main.go -o /path/of/output/dir
 ```
 
+
 ### Setting up CLI
 
-The best way to connect to DiceDB is using [DiceDB CLI](https://github.com/DiceDB/dicedb-cli) and you can install it by running the following command
+The best way to connect to DiceDB is using DiceDB CLI and you can install it by running the following command
 
 ```bash
 sudo su
 curl -sL https://raw.githubusercontent.com/DiceDB/dicedb-cli/refs/heads/master/install.sh | sh
 ```
 
-If you are working on unsupported OS (as per above script), you can always follow the installation instructions mentioned in the [dicedb/cli](https://github.com/DiceDB/dicedb-cli) repository.
+### Client Compatibility
 
-## Running Tests
+DiceDB is fully compatible with Redis protocol, allowing you to connect using any existing Redis client or SDK.
+
+> [!NOTE]
+> The `.WATCH` feature is only accessible through the DiceDB CLI.
+> If you are working on unsupported OS (as per above script), you can always follow the installation instructions mentioned in the [dicedb/cli](https://github.com/DiceDB/dicedb-cli) repository.
+
+### Running Tests
 
 Unit tests and integration tests are essential for ensuring correctness and in the case of DiceDB, both types of tests are available to validate its functionality.
 
@@ -195,8 +189,7 @@ make test
 ## Running Benchmark
 
 ```bash
-go test -test.bench <pattern>
-go test -test.bench BenchmarkListRedis -benchmem
+make run_benchmark
 ```
 
 ## Getting Started
@@ -205,9 +198,6 @@ To get started with building and contributing to DiceDB, please refer to the [is
 
 ## Docs
 
-[![Netlify Status](https://api.netlify.com/api/v1/badges/3a298f6d-ae8d-44d4-a96d-00096b144b55/deploy-status)](https://app.netlify.com/sites/dicedb/deploys)
-[![Built with Starlight](https://astro.badg.es/v2/built-with-starlight/tiny.svg)](https://starlight.astro.build)
-
 We use [Astro](https://astro.build/) framework to power the [dicedb.io website](https://dicedb.io) and [Starlight](https://starlight.astro.build/) to power the docs. Once you have NodeJS installed, fire the following commands to get your local version of [dicedb.io](https://dicedb.io) running.
 
 ```bash
@@ -218,46 +208,31 @@ npm run dev
 
 Once the server starts, visit http://localhost:4321/ in your favourite browser. This runs with a hot reload which means any changes you make in the website and the documentation can be instantly viewed on the browser.
 
-## To build and deploy
-
-```bash
-cd docs
-npm run build
-```
-
 ### Docs directory structure
 
 1. `docs/src/content/docs/commands` is where all the commands are documented
 2. `docs/src/content/docs/tutorials` is where all the tutorials are documented
 
-## The story
+## The Story
 
-DiceDB started as a re-implementation of Redis in Golang and the idea was to - build a DB from scratch and understand the micro-nuances that come with its implementation. The database does not aim to replace Redis, instead, it will fit in and optimize itself for multicore computations running on a single-threaded event loop.
+DiceDB started as a re-implementation of Redis in Golang with the idea of building a DB from scratch to understand the micro-nuances that come with its implementation. DiceDB isn’t just another database; it’s a platform purpose-built for the real-time era. As real-time systems become increasingly prevalent in modern applications, DiceDB’s hyper-optimized architecture is positioned to power the next generation of user experiences.
 
 ## How to contribute
 
 The Code Contribution Guidelines are published at [CONTRIBUTING/README.md](CONTRIBUTING/README.md); please read them before you start making any changes. This would allow us to have a consistent standard of coding practices and developer experience.
 
 Contributors can join the [Discord Server](https://discord.gg/6r8uXWtXh7) for quick collaboration.
 
-## Community
-
-- [Join the Discord Server](https://discord.gg/6r8uXWtXh7)
-
 ## Contributors
 
 <a href = "https://github.com/dicedb/dice/graphs/contributors">
   <img src = "https://contrib.rocks/image?repo=dicedb/dice"/>
 </a>
 
 ## Troubleshoot
-
 ### Forcefully killing the process
 
 ```
 $ sudo netstat -atlpn | grep :7379
 $ sudo kill -9 <process_id>
-```
-
-[discord]: https://discord.gg/6r8uXWtXh7
-[discord-badge]: https://img.shields.io/discord/1034342738960855120?color=%235865F2&label=%20&logo=discord&logoColor=white&style=flat-square
+```
diff --git a/build_protos.sh b/build_protos.sh
@@ -0,0 +1 @@
+protoc --go_out=. ./internal/wal/wal.proto
diff --git a/config/config.go b/config/config.go
@@ -22,12 +22,10 @@ const (
 	DefaultConfigName     string = "dice.toml"
 	DefaultConfigFilePath string = "./"
 
-	EvictSimpleFirst   = "simple-first"
-	EvictAllKeysRandom = "allkeys-random"
-	EvictAllKeysLRU    = "allkeys-lru"
-	EvictAllKeysLFU    = "allkeys-lfu"
+	EvictBatchKeysLRU string = "batch_keys_lru"
 
-	DefaultKeysLimit int = 200000000
+	DefaultKeysLimit     int     = 200000000
+	DefaultEvictionRatio float64 = 0.1
 )
 
 var (
@@ -50,11 +48,17 @@ var (
 
 	InitConfigCmd = false
 
-	KeysLimit = DefaultKeysLimit
+	KeysLimit     = DefaultKeysLimit
+	EvictionRatio = DefaultEvictionRatio
 
 	EnableProfiling = false
 
 	EnableWatch = true
+	LogDir      = ""
+
+	EnableWAL      = true
+	RestoreFromWAL = false
+	WALEngine      = "sqlite"
 )
 
 type Config struct {
@@ -92,11 +96,11 @@ type Config struct {
 	} `mapstructure:"performance"`
 
 	Memory struct {
-		MaxMemory      int64   `mapstructure:"maxmemory"`
-		EvictionPolicy string  `mapstructure:"evictionpolicy"`
-		EvictionRatio  float64 `mapstructure:"evictionratio"`
-		KeysLimit      int     `mapstructure:"keyslimit"`
-		LFULogFactor   int     `mapstructure:"lfulogfactor"`
+		MaxMemory        int64   `mapstructure:"maxmemory"`
+		EvictionStrategy string  `mapstructure:"evictionstrategy"`
+		EvictionRatio    float64 `mapstructure:"evictionratio"`
+		KeysLimit        int     `mapstructure:"keyslimit"`
+		LFULogFactor     int     `mapstructure:"lfulogfactor"`
 	} `mapstructure:"memory"`
 
 	Persistence struct {
@@ -176,17 +180,17 @@ var baseConfig = Config{
 		AdhocReqChanBufSize:    20, // assuming we wouldn't have more than 20 adhoc requests being sent at a time.
 	},
 	Memory: struct {
-		MaxMemory      int64   `mapstructure:"maxmemory"`
-		EvictionPolicy string  `mapstructure:"evictionpolicy"`
-		EvictionRatio  float64 `mapstructure:"evictionratio"`
-		KeysLimit      int     `mapstructure:"keyslimit"`
-		LFULogFactor   int     `mapstructure:"lfulogfactor"`
+		MaxMemory        int64   `mapstructure:"maxmemory"`
+		EvictionStrategy string  `mapstructure:"evictionstrategy"`
+		EvictionRatio    float64 `mapstructure:"evictionratio"`
+		KeysLimit        int     `mapstructure:"keyslimit"`
+		LFULogFactor     int     `mapstructure:"lfulogfactor"`
 	}{
-		MaxMemory:      0,
-		EvictionPolicy: EvictAllKeysLFU,
-		EvictionRatio:  0.9,
-		KeysLimit:      DefaultKeysLimit,
-		LFULogFactor:   10,
+		MaxMemory:        0,
+		EvictionStrategy: EvictBatchKeysLRU,
+		EvictionRatio:    DefaultEvictionRatio,
+		KeysLimit:        DefaultKeysLimit,
+		LFULogFactor:     10,
 	},
 	Persistence: struct {
 		AOFFile            string `mapstructure:"aoffile"`
@@ -361,6 +365,10 @@ func mergeFlagsWithConfig() {
 	if KeysLimit != DefaultKeysLimit {
 		DiceConfig.Memory.KeysLimit = KeysLimit
 	}
+
+	if EvictionRatio != DefaultEvictionRatio {
+		DiceConfig.Memory.EvictionRatio = EvictionRatio
+	}
 }
 
 // This function checks if the config file is present or not at default location or at -c flag location

diff --git a/dice.toml b/dice.toml
@@ -35,7 +35,7 @@ AdhocReqChanBufSize = 20
 [Memory]
 MaxMemory = 0
 EvictionPolicy = 'allkeys-lfu'
-EvictionRatio = 0.9
+EvictionRatio = 0.1
 KeysLimit = 200000000
 LFULogFactor = 10