diff --git a/poetry.lock b/poetry.lock
index a675335667..658289a2e3 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -835,6 +835,17 @@ files = [
 [package.extras]
 tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
 
+[[package]]
+name = "farama-notifications"
+version = "0.0.4"
+description = "Notifications for all Farama Foundation maintained libraries."
+optional = false
+python-versions = "*"
+files = [
+    {file = "Farama-Notifications-0.0.4.tar.gz", hash = "sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18"},
+    {file = "Farama_Notifications-0.0.4-py3-none-any.whl", hash = "sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae"},
+]
+
 [[package]]
 name = "fastjsonschema"
 version = "2.19.1"
@@ -1069,43 +1080,35 @@ docs = ["Sphinx", "furo"]
 test = ["objgraph", "psutil"]
 
 [[package]]
-name = "gym"
-version = "0.26.2"
-description = "Gym: A universal API for reinforcement learning environments"
+name = "gymnasium"
+version = "0.29.1"
+description = "A standard API for reinforcement learning and a diverse set of reference environments (formerly Gym)."
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.8"
 files = [
-    {file = "gym-0.26.2.tar.gz", hash = "sha256:e0d882f4b54f0c65f203104c24ab8a38b039f1289986803c7d02cdbe214fbcc4"},
+    {file = "gymnasium-0.29.1-py3-none-any.whl", hash = "sha256:61c3384b5575985bb7f85e43213bcb40f36fcdff388cae6bc229304c71f2843e"},
+    {file = "gymnasium-0.29.1.tar.gz", hash = "sha256:1a532752efcb7590478b1cc7aa04f608eb7a2fdad5570cd217b66b6a35274bb1"},
 ]
 
 [package.dependencies]
 cloudpickle = ">=1.2.0"
-gym_notices = ">=0.0.4"
-importlib_metadata = {version = ">=4.8.0", markers = "python_version < \"3.10\""}
-numpy = ">=1.18.0"
+farama-notifications = ">=0.0.1"
+importlib-metadata = {version = ">=4.8.0", markers = "python_version < \"3.10\""}
+numpy = ">=1.21.0"
+typing-extensions = ">=4.3.0"
 
 [package.extras]
 accept-rom-license = ["autorom[accept-rom-license] (>=0.4.2,<0.5.0)"]
-all = ["ale-py (>=0.8.0,<0.9.0)", "box2d-py (==2.3.5)", "imageio (>=2.14.1)", "lz4 (>=3.1.0)", "matplotlib (>=3.0)", "moviepy (>=1.0.0)", "mujoco (==2.2)", "mujoco_py (>=2.1,<2.2)", "opencv-python (>=3.0)", "pygame (==2.1.0)", "pytest (==7.0.1)", "swig (==4.*)"]
-atari = ["ale-py (>=0.8.0,<0.9.0)"]
-box2d = ["box2d-py (==2.3.5)", "pygame (==2.1.0)", "swig (==4.*)"]
-classic-control = ["pygame (==2.1.0)"]
-mujoco = ["imageio (>=2.14.1)", "mujoco (==2.2)"]
-mujoco-py = ["mujoco_py (>=2.1,<2.2)"]
-other = ["lz4 (>=3.1.0)", "matplotlib (>=3.0)", "moviepy (>=1.0.0)", "opencv-python (>=3.0)"]
-testing = ["box2d-py (==2.3.5)", "imageio (>=2.14.1)", "lz4 (>=3.1.0)", "matplotlib (>=3.0)", "moviepy (>=1.0.0)", "mujoco (==2.2)", "mujoco_py (>=2.1,<2.2)", "opencv-python (>=3.0)", "pygame (==2.1.0)", "pytest (==7.0.1)", "swig (==4.*)"]
-toy-text = ["pygame (==2.1.0)"]
-
-[[package]]
-name = "gym-notices"
-version = "0.0.8"
-description = "Notices for gym"
-optional = false
-python-versions = "*"
-files = [
-    {file = "gym-notices-0.0.8.tar.gz", hash = "sha256:ad25e200487cafa369728625fe064e88ada1346618526102659b4640f2b4b911"},
-    {file = "gym_notices-0.0.8-py3-none-any.whl", hash = "sha256:e5f82e00823a166747b4c2a07de63b6560b1acb880638547e0cabf825a01e463"},
-]
+all = ["box2d-py (==2.3.5)", "cython (<3)", "imageio (>=2.14.1)", "jax (>=0.4.0)", "jaxlib (>=0.4.0)", "lz4 (>=3.1.0)", "matplotlib (>=3.0)", "moviepy (>=1.0.0)", "mujoco (>=2.3.3)", "mujoco-py (>=2.1,<2.2)", "opencv-python (>=3.0)", "pygame (>=2.1.3)", "shimmy[atari] (>=0.1.0,<1.0)", "swig (==4.*)", "torch (>=1.0.0)"]
+atari = ["shimmy[atari] (>=0.1.0,<1.0)"]
+box2d = ["box2d-py (==2.3.5)", "pygame (>=2.1.3)", "swig (==4.*)"]
+classic-control = ["pygame (>=2.1.3)", "pygame (>=2.1.3)"]
+jax = ["jax (>=0.4.0)", "jaxlib (>=0.4.0)"]
+mujoco = ["imageio (>=2.14.1)", "mujoco (>=2.3.3)"]
+mujoco-py = ["cython (<3)", "cython (<3)", "mujoco-py (>=2.1,<2.2)", "mujoco-py (>=2.1,<2.2)"]
+other = ["lz4 (>=3.1.0)", "matplotlib (>=3.0)", "moviepy (>=1.0.0)", "opencv-python (>=3.0)", "torch (>=1.0.0)"]
+testing = ["pytest (==7.1.3)", "scipy (>=1.7.3)"]
+toy-text = ["pygame (>=2.1.3)", "pygame (>=2.1.3)"]
 
 [[package]]
 name = "h11"
@@ -5138,4 +5141,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "44171f65a1084b6f80ee723431af4c5bffd1f4094599d2bf13d386af38ecfde6"
+content-hash = "df98d6f87b78c451f52e32f91ee2e3aaa145138a78cf500596a7797689a0d3c0"
diff --git a/pyproject.toml b/pyproject.toml
index e3ed735f78..b45f37e632 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@ pandas = "^2.1"
 
 [tool.poetry.group.dev.dependencies]
 graphviz = "^0.20.1"
-gym = "^0.26.2"
+gymnasium = "^0.29.0"
 matplotlib = "^3.0.2"
 mypy = "^1.6.1"
 pre-commit = "^3.5.0"
diff --git a/river/bandit/bayes_ucb.py b/river/bandit/bayes_ucb.py
index 4c62b59d8e..5fdef8d1db 100644
--- a/river/bandit/bayes_ucb.py
+++ b/river/bandit/bayes_ucb.py
@@ -28,7 +28,7 @@ class BayesUCB(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import proba
     >>> from river import stats
diff --git a/river/bandit/envs/__init__.py b/river/bandit/envs/__init__.py
index 8f0f511aa6..6c91fa9b71 100644
--- a/river/bandit/envs/__init__.py
+++ b/river/bandit/envs/__init__.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 try:
-    import gym
+    import gymnasium as gym
 
     GYM_INSTALLED = True
 except ImportError:
diff --git a/river/bandit/envs/candy_cane.py b/river/bandit/envs/candy_cane.py
index 421af13c62..f6702e6606 100644
--- a/river/bandit/envs/candy_cane.py
+++ b/river/bandit/envs/candy_cane.py
@@ -2,7 +2,7 @@
 
 import dataclasses
 
-import gym
+import gymnasium as gym
 
 
 @dataclasses.dataclass
@@ -25,7 +25,7 @@ class CandyCaneContest(gym.Env):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import stats
 
     >>> env = gym.make('river_bandits/CandyCaneContest-v0')
diff --git a/river/bandit/envs/testbed.py b/river/bandit/envs/testbed.py
index c7f0bfb8da..7b8b725d3f 100644
--- a/river/bandit/envs/testbed.py
+++ b/river/bandit/envs/testbed.py
@@ -2,7 +2,7 @@
 
 import math
 
-import gym
+import gymnasium as gym
 
 
 class KArmedTestbed(gym.Env):
diff --git a/river/bandit/epsilon_greedy.py b/river/bandit/epsilon_greedy.py
index e8f52559bd..a422bd1daf 100644
--- a/river/bandit/epsilon_greedy.py
+++ b/river/bandit/epsilon_greedy.py
@@ -33,7 +33,7 @@ class EpsilonGreedy(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import stats
 
diff --git a/river/bandit/evaluate.py b/river/bandit/evaluate.py
index 6baf22b19c..0079079c8e 100644
--- a/river/bandit/evaluate.py
+++ b/river/bandit/evaluate.py
@@ -5,7 +5,7 @@
 import typing
 
 try:
-    import gym
+    import gymnasium as gym
 except ImportError:
     ...
 
@@ -52,7 +52,7 @@ def evaluate(
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
 
     >>> trace = bandit.evaluate(
diff --git a/river/bandit/exp3.py b/river/bandit/exp3.py
index 00e8fb62ca..b5c7899c51 100644
--- a/river/bandit/exp3.py
+++ b/river/bandit/exp3.py
@@ -35,7 +35,7 @@ class Exp3(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import proba
     >>> from river import stats
diff --git a/river/bandit/random.py b/river/bandit/random.py
index 4e7f1c4de1..2199ed0f43 100644
--- a/river/bandit/random.py
+++ b/river/bandit/random.py
@@ -23,7 +23,7 @@ class RandomPolicy(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import proba
     >>> from river import stats
diff --git a/river/bandit/test_envs.py b/river/bandit/test_envs.py
index 0f79fa5759..9be9acf97b 100644
--- a/river/bandit/test_envs.py
+++ b/river/bandit/test_envs.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-import gym.utils.env_checker
+import gymnasium as gym
+import gymnasium.utils.env_checker
 import pytest
 
 from river import bandit
diff --git a/river/bandit/test_policies.py b/river/bandit/test_policies.py
index 9650f488e8..b1d6507ebf 100644
--- a/river/bandit/test_policies.py
+++ b/river/bandit/test_policies.py
@@ -5,7 +5,7 @@
 import inspect
 import random
 
-import gym
+import gymnasium as gym
 import pytest
 
 from river import bandit, metrics
@@ -111,7 +111,7 @@ def test_better_than_random_policy(policy: bandit.base.Policy, env: gym.Env):
             arm_id = policy.pull(arm_ids)  # type: ignore
             observation, reward, terminated, truncated, info = env.step(arm_id)
             policy.update(arm_id, reward)
-            policy_reward += reward
+            policy_reward += float(reward)
 
             random_arm_id = random_policy.pull(arm_ids)  # type: ignore
             (
@@ -122,7 +122,7 @@ def test_better_than_random_policy(policy: bandit.base.Policy, env: gym.Env):
                 info,
             ) = random_env.step(random_arm_id)
             random_policy.update(random_arm_id, reward)
-            random_reward += reward
+            random_reward += float(reward)
 
         n_successes += policy_reward > random_reward
 
diff --git a/river/bandit/thompson.py b/river/bandit/thompson.py
index 03cf76a5d7..c53533448b 100644
--- a/river/bandit/thompson.py
+++ b/river/bandit/thompson.py
@@ -40,7 +40,7 @@ class ThompsonSampling(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import proba
     >>> from river import stats
diff --git a/river/bandit/ucb.py b/river/bandit/ucb.py
index e79f40de51..78fbf0b758 100644
--- a/river/bandit/ucb.py
+++ b/river/bandit/ucb.py
@@ -32,7 +32,7 @@ class UCB(bandit.base.Policy):
     Examples
     --------
 
-    >>> import gym
+    >>> import gymnasium as gym
     >>> from river import bandit
     >>> from river import preprocessing
     >>> from river import stats