From a70721819935b378bc6ffa6051cd6128a30ed0c0 Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Fri, 5 Jul 2024 10:15:23 +0200 Subject: [PATCH] Docs: Add `robots.txt` to only allow indexing of `latest` and `stable` (#6517) Currently, all versions of the documentation are indexed with the result that google searches come up with very outdated versions and the latest version is almost impossible to find. The `robots.txt` now disallows any path from being indexed except for the `latest` and `stable` versions of the documentation. --- docs/source/conf.py | 3 +++ docs/source/robots.txt | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index d017051cb3..03922a0efa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -171,6 +171,9 @@ # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False +# This is to tell search engines to index only stable and latest version +html_extra_path = ['robots.txt'] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/robots.txt b/docs/source/robots.txt new file mode 100644 index 0000000000..f94eda030d --- /dev/null +++ b/docs/source/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: /projects/aiida-core/en/latest/ +Allow: /projects/aiida-core/en/stable/ +Disallow: /