diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 517b813db..81d604296 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,7 +39,7 @@ jobs:
         AutoROM --accept-license --source-file Roms.tar.gz
 
         # cpu version of pytorch - faster to download
-        pip install torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu
+        pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cpu
 
         pip install -r requirements.txt
         # Use headless version
diff --git a/.github/workflows/trained_agents.yml b/.github/workflows/trained_agents.yml
index 689681fe8..a171815c2 100644
--- a/.github/workflows/trained_agents.yml
+++ b/.github/workflows/trained_agents.yml
@@ -40,7 +40,7 @@ jobs:
         AutoROM --accept-license --source-file Roms.tar.gz
 
         # cpu version of pytorch - faster to download
-        pip install torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu
+        pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cpu
         pip install -r requirements.txt
         # Use headless version
         pip install opencv-python-headless
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f6b70dc4..5c41d3ccb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,8 @@
-## Release 2.4.0a0 (WIP)
+## Release 2.4.0a4 (WIP)
 
 ### Breaking Changes
 - Updated defaults hyperparameters for TQC/SAC for Swimmer-v4 (decrease gamma for more consistent results) (@JacobHA) [W&B report](https://wandb.ai/openrlbenchmark/sbx/reports/SAC-MuJoCo-Swimmer-v4--Vmlldzo3NzM5OTk2)
+- Upgraded to SB3 >= 2.4.0
 
 ### New Features
 - Added `CrossQ` hyperparameters for SB3-contrib
@@ -11,7 +12,7 @@
 ### Documentation
 
 ### Other
-
+- Updated PyTorch version to 2.3.1 in the CI
 
 ## Release 2.3.0 (2024-03-31)
 
diff --git a/Makefile b/Makefile
index 31b64750c..1dafee470 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@ lint:
 	# see https://www.flake8rules.com/
 	ruff check ${LINT_PATHS} --select=E9,F63,F7,F82 --output-format=full
 	# exit-zero treats all errors as warnings.
-	ruff check ${LINT_PATHS} --exit-zero
+	ruff check ${LINT_PATHS} --exit-zero --output-format=concise
 
 format:
 	# Sort imports
diff --git a/requirements.txt b/requirements.txt
index 731d1ad53..13ba07b19 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 gym==0.26.2
-stable-baselines3[extra_no_roms,tests,docs]>=2.3.0,<3.0
+stable-baselines3[extra_no_roms,tests,docs]>=2.4.0a0,<3.0
 box2d-py==2.3.8
 pybullet_envs_gymnasium>=0.4.0
 # minigrid
diff --git a/rl_zoo3/version.txt b/rl_zoo3/version.txt
index e96f44fb3..2d22b1587 100644
--- a/rl_zoo3/version.txt
+++ b/rl_zoo3/version.txt
@@ -1 +1 @@
-2.4.0a0
+2.4.0a4
diff --git a/setup.py b/setup.py
index be762639f..5d68ed4d1 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@
 See https://github.com/DLR-RM/rl-baselines3-zoo
 """
 install_requires = [
-    "sb3_contrib>=2.3.0,<3.0",
+    "sb3_contrib>=2.4.0a4,<3.0",
     "gymnasium~=0.29.1",
     "huggingface_sb3>=3.0,<4.0",
     "tqdm",
@@ -24,7 +24,8 @@
     "pyyaml>=5.1",
     "pytablewriter~=1.2",
 ]
-plots_requires = ["seaborn", "rliable>=1.0.5", "scipy~=1.10"]
+# TODO(antonin): update to rliable>=1.1.0 once PR is merged and released
+plots_requires = ["seaborn", "rliable @ git+https://github.com/araffin/rliable@patch-1", "scipy~=1.10"]
 test_requires = [
     # for MuJoCo envs v4:
     "mujoco~=2.3",