lsp: Use fallback env when possible

If a client provides the `esbonio.sphinx.fallbackEnv`, repurpose the server's Python interpreter
to launch the Sphinx agent.

lib/esbonio/esbonio/server/features/sphinx_manager/config.py

@@ -3,6 +3,7 @@
 import importlib.util
 import logging
 import pathlib
+import sys
 from typing import Any
 from typing import Optional
 
@@ -60,6 +61,12 @@ class SphinxConfig:
     cwd: str = attrs.field(default="${scopeFsPath}")
     """The working directory to use."""
 
+    fallback_env: str | None = attrs.field(default=None)
+    """Location of the fallback environment to use.
+
+    Intended to be used by clients to handle the case where the user has not configured
+    ``python_command`` themselves."""
+
     python_path: list[pathlib.Path] = attrs.field(factory=list)
     """The value of ``PYTHONPATH`` to use when injecting the sphinx agent into the
     target environment"""
@@ -89,8 +96,8 @@ def resolve(
            The fully resolved config object to use.
            If ``None``, a valid configuration could not be created.
         """
-        python_path = self._resolve_python_path(logger)
-        if len(python_path) == 0:
+        python_command, python_path = self._resolve_python(logger)
+        if len(python_path) == 0 or len(python_command) == 0:
             return None
 
         cwd = self._resolve_cwd(uri, workspace, logger)
@@ -109,7 +116,7 @@ def resolve(
             config_overrides=self.config_overrides,
             cwd=cwd,
             env_passthrough=self.env_passthrough,
-            python_command=self.python_command,
+            python_command=python_command,
             build_command=build_command,
             python_path=python_path,
         )
@@ -158,12 +165,25 @@ def _resolve_cwd(
 
         return None
 
-    def _resolve_python_path(self, logger: logging.Logger) -> list[pathlib.Path]:
-        """Return the list of paths to put on the sphinx agent's ``PYTHONPATH``
+    def _resolve_python(
+        self, logger: logging.Logger
+    ) -> tuple[list[str], list[pathlib.Path]]:
+        """Return the python configuration to use when launching the sphinx agent.
+
+        The first element of the returned tuple is the command to use when running the
+        sphinx agent. This could be as simple as the path to the python interpreter in a
+        particular virtual environment or a complex command such as
+        ``hatch -e docs run python``.
 
         Using the ``PYTHONPATH`` environment variable, we can inject additional Python
-        packages into the user's Python environment. This method will locate the
-        installation path of the sphinx agent and return it.
+        packages into the user's Python environment. This method also locates the
+        installation path of the sphinx agent and returns it in the second element of the
+        tuple.
+
+        Finally, if the user has not configured a python environment and the client has
+        set the ``fallback_env`` option, this method will construct a command based on
+        the current interpreter to create an isolated environment based on
+        ``fallback_env``.
 
         Parameters
         ----------
@@ -172,19 +192,34 @@ def _resolve_python_path(self, logger: logging.Logger) -> list[pathlib.Path]:
 
         Returns
         -------
-        List[pathlib.Path]
-           The list of paths to Python packages to inject into the sphinx agent's target
-           environment. If empty, the ``esbonio.sphinx_agent`` package was not found.
+        tuple[list[str], list[pathlib.Path]]
+           A tuple of the form ``(python_command, python_path)``.
         """
-        if len(self.python_path) > 0:
-            return self.python_path
-
-        if (sphinx_agent := get_module_path("esbonio.sphinx_agent")) is None:
-            logger.error("Unable to locate the sphinx agent")
-            return []
-
-        python_path = [sphinx_agent]
-        return python_path
+        if len(python_path := list(self.python_path)) == 0:
+            if (sphinx_agent := get_module_path("esbonio.sphinx_agent")) is None:
+                logger.error("Unable to locate the sphinx agent")
+                return [], []
+
+            python_path.append(sphinx_agent)
+
+        if len(python_command := list(self.python_command)) == 0:
+            if self.fallback_env is None:
+                logger.error("No python command configured")
+                return [], []
+
+            if not (fallback_env := pathlib.Path(self.fallback_env)).exists():
+                logger.error(
+                    "Provided fallback environment %s does not exist", fallback_env
+                )
+                return [], []
+
+            # Since the client has provided a fallback environment we can isolate the
+            # current Python interpreter from its environment and reuse it.
+            logger.debug("Using fallback environment")
+            python_path.append(fallback_env)
+            python_command.extend([sys.executable, "-S"])
+
+        return python_command, python_path
 
     def _resolve_build_command(self, uri: Uri, logger: logging.Logger) -> list[str]:
         """Return the ``sphinx-build`` command to use.

lib/esbonio/tests/e2e/test_sphinx_manager.py

@@ -138,9 +138,17 @@ async def test_get_client(
 async def test_get_client_with_error(
     server_manager: ServerManager, demo_workspace: Uri
 ):
-    """Ensure that we correctly handle the case where there is an error with a client."""
+    """Ensure that we correctly handle the case where there is an error with the client."""
 
-    server, manager = server_manager(None)
+    server, manager = server_manager(
+        dict(
+            esbonio=dict(
+                sphinx=dict(
+                    pythonCommand=["/not/a/real/env/python"],
+                ),
+            ),
+        ),
+    )
     # Ensure that the server is ready
     await server.ready
 
@@ -165,7 +173,7 @@ async def test_get_client_with_error(
 
     assert result is client
     assert client.state == ClientState.Errored
-    assert "No python environment configured" in str(client.exception)
+    assert "No such file or directory" in str(client.exception)
 
     # Finally, if we request another uri from the same project we should get back
     # the same client instance - even though it failed to start.

lib/esbonio/tests/server/features/test_sphinx_config.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import pathlib
+import sys
 from typing import Optional
 
 import pytest
@@ -19,6 +20,10 @@
 BUILD_CMD = ["sphinx-build", "-M", "html", "src", "dest"]
 PYPATH = [pathlib.Path("/path/to/site-packages/esbonio")]
 
+# The value of FALLBACK_ENV must actually exist somewhere on the filesystem
+# But for the tests here, the actual location doesn't really matter
+FALLBACK_ENV = str(pathlib.Path(__file__).parent)
+
 
 def mk_uri(path: str) -> str:
     return str(Uri.for_file(path))
@@ -151,6 +156,54 @@ def mk_uri(path: str) -> str:
             ),
             None,
         ),
+        (  # If no python command provided, and no fallback env
+            # available, the configuration is invalid
+            "file:///path/to/workspace/file.rst",
+            Workspace(None),
+            SphinxConfig(
+                python_command=[],
+                build_command=BUILD_CMD,
+                cwd=CWD,
+                python_path=PYPATH,
+            ),
+            None,
+        ),
+        (  # If no python command provided, but there is a fallback env
+            # use that
+            "file:///path/to/workspace/file.rst",
+            Workspace(None),
+            SphinxConfig(
+                python_command=[],
+                fallback_env=FALLBACK_ENV,
+                build_command=BUILD_CMD,
+                cwd=CWD,
+                python_path=PYPATH,
+            ),
+            SphinxConfig(
+                python_command=[sys.executable, "-S"],
+                build_command=BUILD_CMD,
+                cwd=CWD,
+                python_path=[PYPATH[0], pathlib.Path(FALLBACK_ENV)],
+            ),
+        ),
+        (  # If a fallback_env is available, but the user has provided their
+            # own python_command, we should really use that.
+            "file:///path/to/workspace/file.rst",
+            Workspace(None),
+            SphinxConfig(
+                python_command=PYTHON_CMD,
+                fallback_env=FALLBACK_ENV,
+                build_command=BUILD_CMD,
+                cwd=CWD,
+                python_path=PYPATH,
+            ),
+            SphinxConfig(
+                python_command=PYTHON_CMD,
+                build_command=BUILD_CMD,
+                cwd=CWD,
+                python_path=PYPATH,
+            ),
+        ),
     ],
 )
 def test_resolve(