Load session using cabal's --with-repl command

This avoids quite a lot of hacky implementations to find the correct GHC
version, libdir and ghc-pkg location, as well as global package db
stuff.

There is a corner case which we need to handle:

```
cabal >= 3.15 && lib:Cabal <3.15
```

In other words, when a custom-setup imposes a restriction on the
`lib:Cabal` version, and this `lib:Cabal` version doesn't support the
`--with-repl` flag, then we can't use `--with-repl`.

We detect this by looking at the error message of the cabal command, it
looks roughly like this:

    Error: [Cabal-7107]
    Could not resolve dependencies:
    [__0] trying: cabal-with-custom-setup-0.1.0.0 (user goal)
    [__1] next goal: cabal-with-custom-setup:setup.Cabal (dependency of cabal-with-custom-setup)
    [__1] rejecting: cabal-with-custom-setup:setup.Cabal; 3.10.3.0/installed-3.10.3.0, ... (constraint from --with-repl requires >=3.15)
    ...

We check the error message for the existence of `constraint from
--with-repl requires >=3.15` and then fall back to using the old
approach which doesn't use response files or `--with-repl` flag.

Additionally, we extract a new `Process` module which contains helpers
for cradle actions.

Author: fendor

.github/workflows/haskell.yml

@@ -15,17 +15,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cabal: ['3.14.1.0', '3.10.2.0']
-        ghc: ['9.12.2', '9.10.1', '9.8.4', '9.6.7', '9.4.8']
+        cabal: ['head', '3.14.1.0', '3.10.2.0']
+        ghc: ['9.12.2', '9.10.1', '9.8.4', '9.6.7']
         os: [ubuntu-latest, macOS-latest, windows-latest]
+        exclude:
+          - cabal: 'head'
+            os: macOS-latest
+          - cabal: '3.10.2.0'
+            ghc: '9.12.2'
 
     steps:
     - uses: actions/checkout@v4
     - id:   extra-ghc
       uses: haskell-actions/setup@v2
       with:
         cabal-version: ${{ matrix.cabal }}
-        ghc-version: '9.2.8' # This needs to be in-sync with 'extraGhcVersion'
+        ghc-version: '9.4.8' # This needs to be in-sync with 'extraGhcVersion'
 
     - uses: haskell-actions/setup@v2
       with:
@@ -35,7 +40,7 @@ jobs:
 
     - name: Print extra ghc version
       # Needs to be in sync with 'extraGhcVersion'
-      run: ghc-9.2.8 --version
+      run: ghc-9.4.8 --version
 
     - name: Print normal ghc version
       run: ghc --version
@@ -93,3 +98,6 @@ jobs:
         ghc-version: ${{ matrix.ghc }}
     - name: Compile Windows wrapper
       run: ghc -hide-all-packages -package base -package directory -package process -Wall -Werror wrappers/cabal.hs -o CabalWrapper
+
+    - name: Compile `--with-repl` Windows wrapper
+      run: ghc -hide-all-packages -package base -package directory -package process -Wall -Werror wrappers/cabal-with-repl.hs -o CabalWithReplWrapper

hie-bios.cabal

@@ -15,18 +15,20 @@ Build-Type:             Simple
 extra-doc-files:        ChangeLog.md
 Extra-Source-Files:     README.md
                         wrappers/cabal
+                        wrappers/cabal-with-repl
                         wrappers/cabal.hs
+                        wrappers/cabal-with-repl.hs
                         tests/configs/*.yaml
                         tests/projects/cabal-with-ghc/cabal-with-ghc.cabal
                         tests/projects/cabal-with-ghc/cabal.project
                         tests/projects/cabal-with-ghc/hie.yaml
                         tests/projects/cabal-with-ghc/src/MyLib.hs
                         tests/projects/cabal-with-ghc-and-project/cabal-with-ghc.cabal
-                        tests/projects/cabal-with-ghc-and-project/cabal.project.9.2.8
+                        tests/projects/cabal-with-ghc-and-project/cabal.project.extra
                         tests/projects/cabal-with-ghc-and-project/hie.yaml
                         tests/projects/cabal-with-ghc-and-project/src/MyLib.hs
                         tests/projects/cabal-with-project/cabal-with-project.cabal
-                        tests/projects/cabal-with-project/cabal.project.9.2.8
+                        tests/projects/cabal-with-project/cabal.project.extra
                         tests/projects/cabal-with-project/hie.yaml
                         tests/projects/cabal-with-project/src/MyLib.hs
                         tests/projects/symlink-test/a/A.hs
@@ -47,7 +49,7 @@ Extra-Source-Files:     README.md
                         tests/projects/multi-cabal-with-project/appA/src/Lib.hs
                         tests/projects/multi-cabal-with-project/appB/appB.cabal
                         tests/projects/multi-cabal-with-project/appB/src/Lib.hs
-                        tests/projects/multi-cabal-with-project/cabal.project.9.2.8
+                        tests/projects/multi-cabal-with-project/cabal.project.extra
                         tests/projects/multi-cabal-with-project/hie.yaml
                         tests/projects/monorepo-cabal/cabal.project
                         tests/projects/monorepo-cabal/hie.yaml
@@ -144,7 +146,7 @@ Extra-Source-Files:     README.md
                         tests/projects/failing-multi-repl-cabal-project/multi-repl-cabal-fail/src/Lib.hs
                         tests/projects/failing-multi-repl-cabal-project/NotInPath.hs
 
-tested-with: GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.1 || ==9.12.2
+tested-with: GHC ==9.6.7 || ==9.8.4 || ==9.10.1 || ==9.12.2
 
 Library
   Default-Language:     Haskell2010

src/HIE/Bios/Cradle.hs

@@ -1,7 +1,20 @@
 {-# LANGUAGE BangPatterns #-}
 {-# LANGUAGE TupleSections #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-module HIE.Bios.Process where
+module HIE.Bios.Process
+  ( CreateProcess(..)
+  -- * Run processes with extra environment variables
+  , readProcessWithCwd
+  , readProcessWithCwd_
+  , readProcessWithCwd'
+  , readProcessWithOutputs
+  , getCleanEnvironment
+  -- * Find file utilities
+  , findFileUpwards
+  , findFileUpwardsPredicate
+  , findFile
+  )
+  where
 
 import Control.Applicative (optional)
 import Control.DeepSeq

src/HIE/Bios/Process.hs

@@ -16,7 +16,7 @@ import qualified Control.Monad.Fail as Fail
 import           Data.Maybe (fromMaybe)
 import qualified Data.Text as T
 import           Prettyprinter
-import           System.Process.Extra (CreateProcess (env, cmdspec), CmdSpec (..))
+import           System.Process.Extra (CreateProcess (env, cmdspec), CmdSpec (..), showCommandForUser)
 
 ----------------------------------------------------------------
 -- Environment variables used by hie-bios.
@@ -106,6 +106,7 @@ data Log
   | LogComputedCradleLoadStyle !T.Text !LoadStyle
   | LogLoadWithContextUnsupported !T.Text !(Maybe T.Text)
   | LogCabalLoad !FilePath !(Maybe String) ![FilePath] ![FilePath]
+  | LogCabalLibraryTooOld [String]
   | LogCabalPath !T.Text
   deriving (Show)
 
@@ -148,6 +149,9 @@ instance Pretty Log where
           <> line <> indent 4 "from project: " <+> pretty projectFile
           <> line <> indent 4 "with prefixes:" <+> pretty prefixes
           <> line <> indent 4 "with actual loading files:" <+> pretty crs
+  pretty (LogCabalLibraryTooOld err) =
+    "'lib:Cabal' is too old to use '--with-repl' flag. Requires 'lib:Cabal' >= 3.15. Original error:" <> line
+      <> vcat (fmap pretty err)
   pretty (LogCabalPath err) =
     "Could not parse json output of 'cabal path': "
       <> line <> indent 4 (pretty err)
@@ -316,12 +320,18 @@ data ComponentOptions = ComponentOptions {
 -- | Prettify 'CmdSpec', so we can show the command to a user
 prettyCmdSpec :: CmdSpec -> String
 prettyCmdSpec (ShellCommand s) = s
-prettyCmdSpec (RawCommand cmd args) = cmd ++ " " ++ unwords args
+prettyCmdSpec (RawCommand cmd args) = showCommandForUser cmd args
 
 -- | Pretty print hie-bios's relevant environment variables.
 prettyProcessEnv :: CreateProcess -> [String]
 prettyProcessEnv p =
   [ key <> ": " <> value
+  | (key, value) <- hieBiosProcessEnv p
+  ]
+
+hieBiosProcessEnv :: CreateProcess -> [(String, String)]
+hieBiosProcessEnv p =
+  [ (key, value)
   | (key, value) <- fromMaybe [] (env p)
   , key `elem` [ hie_bios_output
                , hie_bios_ghc

src/HIE/Bios/Types.hs

@@ -1,17 +1,24 @@
-{-# LANGUAGE TemplateHaskell #-}
 {-# LANGUAGE QuasiQuotes #-}
-{-# LANGUAGE CPP #-}
-module HIE.Bios.Wrappers (cabalWrapper, cabalWrapperHs) where
+{-# LANGUAGE TemplateHaskell #-}
 
-import Data.FileEmbed
-#if __GLASGOW_HASKELL__ >= 903
-  hiding (makeRelativeToProject)
-#endif
-import Language.Haskell.TH.Syntax
+module HIE.Bios.Wrappers (
+  cabalWrapper,
+  cabalWithReplWrapper,
+  cabalWrapperHs,
+  cabalWithReplWrapperHs,
+) where
+
+import Data.FileEmbed ( embedStringFile )
+import Language.Haskell.TH.Syntax ( makeRelativeToProject )
 
 cabalWrapper :: String
 cabalWrapper = $(makeRelativeToProject "wrappers/cabal" >>= embedStringFile)
 
 cabalWrapperHs :: String
 cabalWrapperHs = $(makeRelativeToProject "wrappers/cabal.hs" >>= embedStringFile)
 
+cabalWithReplWrapper :: String
+cabalWithReplWrapper = $(makeRelativeToProject "wrappers/cabal-with-repl" >>= embedStringFile)
+
+cabalWithReplWrapperHs :: String
+cabalWithReplWrapperHs = $(makeRelativeToProject "wrappers/cabal-with-repl.hs" >>= embedStringFile)

src/HIE/Bios/Wrappers.hs

@@ -18,9 +18,10 @@ import HIE.Bios.Cradle
 import Control.Monad ( forM_ )
 import Data.List ( sort, isPrefixOf )
 import Data.Typeable
+import System.Exit (ExitCode(ExitSuccess, ExitFailure))
 import System.Directory
 import System.FilePath ((</>), makeRelative)
-import System.Exit (ExitCode(ExitSuccess, ExitFailure))
+import System.Info.Extra (isWindows)
 import Control.Monad.Extra (unlessM)
 import qualified HIE.Bios.Ghc.Gap as Gap
 import Control.Monad.IO.Class
@@ -34,7 +35,7 @@ argDynamic = ["-dynamic" | Gap.hostIsDynamic]
 -- If you change this version, make sure to also update 'cabal.project'
 -- in 'tests\/projects\/cabal-with-ghc'.
 extraGhcVersion :: String
-extraGhcVersion = "9.2.8"
+extraGhcVersion = "9.4.8"
 
 extraGhc :: String
 extraGhc = "ghc-" ++ extraGhcVersion
@@ -123,7 +124,11 @@ biosTestCases =
         cradleErrorExitCode @?= ExitSuccess
         cradleErrorDependencies `shouldMatchList` []
         length cradleErrorStderr @?= 1
-        "Couldn't execute myGhc" `isPrefixOf` head cradleErrorStderr @? "Error message should contain basic information"
+        let errorCtx = head cradleErrorStderr
+        -- On windows, this error message contains '"' around the executable name
+        if isWindows
+          then "Couldn't execute \"myGhc\"" `isPrefixOf` errorCtx @? "Error message should contain error information"
+          else "Couldn't execute myGhc"     `isPrefixOf` errorCtx @? "Error message should contain error information"
   , biosTestCase "simple-bios-shell" $ runTestEnv "./simple-bios-shell" $ do
       testDirectoryM isBiosCradle "B.hs"
   , biosTestCase "simple-bios-shell-deps" $ runTestEnv "./simple-bios-shell" $ do
@@ -242,6 +247,12 @@ cabalTestCases extraGhcDep =
           loadRuntimeGhcVersion
           assertGhcVersionIs extraGhcVersion
       ]
+    , biosTestCase "force older Cabal version in custom setup" $ runTestEnv "cabal-with-custom-setup" $ do
+        -- Specifically tests whether cabal 3.16 works as expected with
+        -- an older lib:Cabal version that doesn't support '--with-repl'.
+        -- This test doesn't hurt for other cases as well, so we enable it for
+        -- all configurations.
+        testDirectoryM isCabalCradle "src/MyLib.hs"
     ]
   ]
   where
@@ -358,7 +369,6 @@ biosTestCase name assertion = askOption @VerboseLogging (\case
   VerboseLogging verbose -> testCase name (assertion verbose)
   )
 
-
 -- ------------------------------------------------------------------
 -- Stack related helper functions
 -- ------------------------------------------------------------------

tests/BiosTests.hs

@@ -62,11 +62,11 @@ main = defaultMain $
                                                                                ,("./tests", StackType (Just "parser-tests") Nothing)]))]))
     -- Assertions for cabal.project files
     assertParser "cabal-with-project.yaml"
-      (noDeps (Cabal $ CabalType Nothing (Just "cabal.project.9.2.8")))
+      (noDeps (Cabal $ CabalType Nothing (Just "cabal.project.extra")))
     assertParser "cabal-with-both.yaml"
-      (noDeps (Cabal $ CabalType (Just "hie-bios:hie") (Just "cabal.project.9.2.8")))
+      (noDeps (Cabal $ CabalType (Just "hie-bios:hie") (Just "cabal.project.extra")))
     assertParser "multi-cabal-with-project.yaml"
-      (noDeps (CabalMulti (CabalType Nothing (Just "cabal.project.9.2.8"))
+      (noDeps (CabalMulti (CabalType Nothing (Just "cabal.project.extra"))
                           [("./src", CabalType (Just "lib:hie-bios") Nothing)
                           ,("./vendor", CabalType (Just "parser-tests") Nothing)]))
     -- Assertions for stack.yaml files

tests/ParserTests.hs

@@ -1,4 +1,4 @@
 cradle:
   cabal:
-    cabalProject: "cabal.project.9.2.8"
+    cabalProject: "cabal.project.extra"
     component: "hie-bios:hie"

tests/configs/cabal-with-both.yaml

@@ -1,3 +1,3 @@
 cradle:
   cabal:
-    cabalProject: "cabal.project.9.2.8"
+    cabalProject: "cabal.project.extra"