diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml
index 202b2b190..d89b25946 100644
--- a/.github/workflows/github-actions.yml
+++ b/.github/workflows/github-actions.yml
@@ -111,10 +111,9 @@ jobs:
         cd docs
         make html-readthedocs
 
-  build-n-publish-dummy:
+  build-dryrun:
     runs-on: ubuntu-latest
     needs: [linting, test, build_doc_dryrun]
-    if: github.event_name != 'pull_request'
     steps:
     - uses: actions/checkout@master
     - uses: actions/setup-python@v4
@@ -124,19 +123,11 @@ jobs:
     - name: build dummy wheel for test-pypi
       run: |
         pip install wheel
-        python setup.py egg_info -b ".dev`date '+%Y%m%d%H%M%S'`" build sdist bdist_wheel
-#    - name: publish test-pypi
-#      # Although working and recommended, test-pypi has a limit
-#      # in the size of projects so it's better to avoid publishing
-#      # until there is a way to garbage collect these dummy releases
-#      uses: pypa/gh-action-pypi-publish@master
-#      with:
-#        password: ${{ secrets.test_pypi_token }}
-#        repository_url: https://test.pypi.org/legacy/
+        python setup.py sdist bdist_wheel
 
   build-n-publish:
     runs-on: ubuntu-latest
-    needs: [linting, test, build_doc_dryrun, build-n-publish-dummy]
+    needs: [linting, test, build_doc_dryrun, build-dryrun]
     if: github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
     steps:
     - uses: actions/checkout@master
diff --git a/docs/changelog.rst b/docs/changelog.rst
index a768ab345..fd39f96e5 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,12 +7,15 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+
+Changes in 0.25.0
+=================
 - Support MONGODB-AWS authentication mechanism (with `authmechanismproperties`) #2507
 - Turning off dereferencing for the results of distinct query. #2663
 - Add tests against Mongo 5.0 in pipeline
 - Drop support for Python 3.6 (EOL)
-- Bug fix support for PyMongo>=4 to fix "InvalidOperation: Cannot use MongoClient after close"
-  errors.
+- Bug fix support for PyMongo>=4 to fix "pymongo.errors.InvalidOperation: Cannot use MongoClient after close"
+  errors. #2627
 
 Changes in 0.24.2
 =================
diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py
index aa38f5b8b..08192b80a 100644
--- a/mongoengine/__init__.py
+++ b/mongoengine/__init__.py
@@ -29,7 +29,7 @@
 )
 
 
-VERSION = (0, 24, 2)
+VERSION = (0, 25, 0)
 
 
 def get_version():
diff --git a/tests/__init__.py b/tests/__init__.py
index e69de29bb..29a312ae0 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -0,0 +1,7 @@
+import os
+
+_THIS_MODULE = os.path.abspath(__file__)
+TESTS_DIR = os.path.dirname(_THIS_MODULE)
+
+ROOT_DIR = os.path.dirname(TESTS_DIR)
+DOCS_DIR = os.path.join(ROOT_DIR, "docs")
diff --git a/tests/test_changelog_consistency.py b/tests/test_changelog_consistency.py
new file mode 100644
index 000000000..c612aa0f2
--- /dev/null
+++ b/tests/test_changelog_consistency.py
@@ -0,0 +1,26 @@
+import os
+from pathlib import Path
+
+from mongoengine import get_version
+from tests import DOCS_DIR
+
+
+def test_package_version_described_in_changelog():
+    """Ensures that changelog is updated when version is incremented"""
+    version_str = get_version()
+    changelog_content = Path(os.path.join(DOCS_DIR, "changelog.rst")).read_text()
+    assert (
+        version_str in changelog_content
+    ), "Version in __init__.py not present in changelog"
+
+
+def test_package_version_incremented_when_new_version_added_to_changelog():
+    """Ensures that changelog is updated when version is incremented"""
+    version_str = get_version()
+    changelog_content = Path(os.path.join(DOCS_DIR, "changelog.rst")).read_text()
+
+    def find_between(s, start, end):
+        return (s.split(start))[1].split(end)[0]
+
+    most_recent_version = find_between(changelog_content, start="Changes in ", end="\n")
+    assert most_recent_version == version_str