feat: Support for optional python_site_packages_path in repodata (#3579)

Co-authored-by: AntoinePrv <[email protected]>
Co-authored-by: Julien Jerphanion <[email protected]>

Author: 3 people

dev/environment-dev.yml

@@ -29,6 +29,7 @@ dependencies:
   # micromamba test dependencies
   - python
   - mitmproxy
+  - bcrypt==4.0.1 # An indirect dependency pin due to failures on MacOS
   - pytest >=7.3.0
   - pytest-asyncio
   - pytest-timeout

libmamba/ext/solv-cpp/include/solv-cpp/solvable.hpp

@@ -53,6 +53,7 @@ namespace solv
         auto build_string() const -> std::string_view;
         auto file_name() const -> std::string_view;
         auto license() const -> std::string_view;
+        auto python_site_packages_path() const -> std::string_view;
         auto md5() const -> std::string_view;
         auto noarch() const -> std::string_view;
         auto sha256() const -> std::string_view;
@@ -168,6 +169,17 @@ namespace solv
         void set_license(raw_str_view str) const;
         void set_license(const std::string& str) const;
 
+        /**
+         * Set the python_site_packages_path of the solvable.
+         *
+         * This is not used by libsolv and is purely for data storing.
+         *
+         * @note A call to @ref ObjRepoView::internalize is required for this attribute to
+         *       be available for lookup.
+         */
+        void set_python_site_packages_path(raw_str_view str) const;
+        void set_python_site_packages_path(const std::string& str) const;
+
         /**
          * Set the md5 hash of the solvable file.
          *

libmamba/ext/solv-cpp/src/solvable.cpp

@@ -206,6 +206,22 @@ namespace solv
         return set_license(str.c_str());
     }
 
+    auto ObjSolvableViewConst::python_site_packages_path() const -> std::string_view
+    {
+        return ptr_to_strview(::solvable_lookup_str(const_cast<::Solvable*>(raw()), SOLVABLE_MEDIABASE)
+        );
+    }
+
+    void ObjSolvableView::set_python_site_packages_path(raw_str_view str) const
+    {
+        ::solvable_set_str(raw(), SOLVABLE_MEDIABASE, str);
+    }
+
+    void ObjSolvableView::set_python_site_packages_path(const std::string& str) const
+    {
+        return set_python_site_packages_path(str.c_str());
+    }
+
     auto ObjSolvableViewConst::md5() const -> std::string_view
     {
         ::Id type = 0;

libmamba/ext/solv-cpp/tests/src/test_solvable.cpp

@@ -43,6 +43,7 @@ namespace
             solv.set_build_string("build");
             solv.set_file_name("file.tar.gz");
             solv.set_license("MIT");
+            solv.set_python_site_packages_path("dummy_pspp");
             solv.set_md5("6f29ba77e8b03b191c9d667f331bf2a0");
             solv.set_sha256("ecde63af23e0d49c0ece19ec539d873ea408a6f966d3126994c6d33ae1b9d3f7");
             solv.set_signatures(
@@ -62,6 +63,7 @@ namespace
                 REQUIRE(solv.build_string() == "");
                 REQUIRE(solv.file_name() == "");
                 REQUIRE(solv.license() == "");
+                REQUIRE(solv.python_site_packages_path() == "");
                 REQUIRE(solv.md5() == "");
                 REQUIRE(solv.sha256() == "");
                 REQUIRE(solv.signatures() == "");
@@ -83,6 +85,7 @@ namespace
                 REQUIRE(solv.build_string() == "build");
                 REQUIRE(solv.file_name() == "file.tar.gz");
                 REQUIRE(solv.license() == "MIT");
+                REQUIRE(solv.python_site_packages_path() == "dummy_pspp");
                 REQUIRE(solv.md5() == "6f29ba77e8b03b191c9d667f331bf2a0");
                 REQUIRE(
                     solv.sha256() == "ecde63af23e0d49c0ece19ec539d873ea408a6f966d3126994c6d33ae1b9d3f7"

libmamba/include/mamba/core/transaction.hpp

@@ -82,7 +82,16 @@ namespace mamba
         History::UserRequest m_history_entry;
         solver::Solution m_solution;
 
+        /** Pair of current Python version, and potential update. */
         std::pair<std::string, std::string> m_py_versions;
+        /**
+         * The potential "python_site_package" entry.
+         *
+         * Found in the the new or installed python interpreter.
+         * Key is added as part of CEP-17.
+         * https://conda.org/learn/ceps/cep-0017
+         */
+        std::string m_python_site_packages_path;
         std::vector<specs::MatchSpec> m_requested_specs;
 
         MTransaction(const CommandParams& command_params, MultiPackageCache&);

libmamba/include/mamba/specs/package_info.hpp

@@ -47,6 +47,7 @@ namespace mamba::specs
         std::string license = {};
         std::string md5 = {};
         std::string sha256 = {};
+        std::string python_site_packages_path = {};
         std::string signatures = {};
         std::vector<std::string> track_features = {};
         std::vector<std::string> dependencies = {};

libmamba/include/mamba/specs/repo_data.hpp

@@ -63,6 +63,9 @@ namespace mamba::specs
         /** Optionally a SHA256 hash of the package archive. */
         std::optional<std::string> sha256 = {};
 
+        /** Optionally a path to the site-packages directory. */
+        std::optional<std::string> python_site_packages_path = {};
+
         /** A deprecated md5 hash. */
         std::optional<std::string> legacy_bz2_md5 = {};
 

libmamba/src/core/query.cpp

@@ -325,6 +325,11 @@ namespace mamba
                 fmt::print(out, fmtstring, "Track Features", fmt::join(pkg.track_features, ","));
             }
 
+            if (!pkg.python_site_packages_path.empty())
+            {
+                fmt::print(out, fmtstring, "Site-packages", pkg.python_site_packages_path);
+            }
+
             // std::cout << fmt::format<char>(
             // " {:<15} {:%Y-%m-%d %H:%M:%S} UTC\n", "Timestamp", fmt::gmtime(pkg.timestamp));
 

libmamba/src/core/transaction.cpp

@@ -27,17 +27,15 @@
 #include "mamba/core/repo_checker_store.hpp"
 #include "mamba/core/thread_utils.hpp"
 #include "mamba/core/transaction.hpp"
-#include "mamba/core/util_os.hpp"
 #include "mamba/solver/libsolv/database.hpp"
 #include "mamba/specs/match_spec.hpp"
-#include "mamba/util/environment.hpp"
 #include "mamba/util/variant_cmp.hpp"
 
-#include "./link.hpp"
-#include "./transaction_context.hpp"
 #include "solver/helpers.hpp"
 
+#include "link.hpp"
 #include "progress_bar_impl.hpp"
+#include "transaction_context.hpp"
 
 namespace mamba
 {
@@ -110,19 +108,22 @@ namespace mamba
             return out;
         }
 
-        auto
-        find_python_version(const solver::Solution& solution, const solver::libsolv::Database& database)
-            -> std::pair<std::string, std::string>
+        auto find_python_versions_and_site_packages(
+            const solver::Solution& solution,
+            const solver::libsolv::Database& database
+        ) -> std::pair<std::pair<std::string, std::string>, std::string>
         {
             // We need to find the python version that will be there after this
             // Transaction is finished in order to compile the noarch packages correctly,
 
             // We need to look into installed packages in case we are not installing a new python
             // version but keeping the current one.
             // Could also be written in term of PrefixData.
+            std::string python_site_packages_path = {};
             std::string installed_py_ver = {};
             if (auto pkg = installed_python(database))
             {
+                python_site_packages_path = pkg->python_site_packages_path;
                 installed_py_ver = pkg->version;
                 LOG_INFO << "Found python in installed packages " << installed_py_ver;
             }
@@ -131,9 +132,13 @@ namespace mamba
             if (auto py = solver::find_new_python_in_solution(solution))
             {
                 new_py_ver = py->get().version;
+                python_site_packages_path = py->get().python_site_packages_path;
             }
 
-            return { std::move(new_py_ver), std::move(installed_py_ver) };
+            return {
+                { std::move(new_py_ver), std::move(installed_py_ver) },
+                std::move(python_site_packages_path),
+            };
         }
     }
 
@@ -213,7 +218,10 @@ namespace mamba
             Console::instance().json_write({ { "PREFIX", ctx.prefix_params.target_prefix.string() } });
         }
 
-        m_py_versions = find_python_version(m_solution, database);
+        std::tie(
+            m_py_versions,
+            m_python_site_packages_path
+        ) = find_python_versions_and_site_packages(m_solution, database);
     }
 
     MTransaction::MTransaction(
@@ -258,7 +266,10 @@ namespace mamba
             [&](const auto& item) { m_requested_specs.push_back(item.spec); }
         );
 
-        m_py_versions = find_python_version(m_solution, database);
+        std::tie(
+            m_py_versions,
+            m_python_site_packages_path
+        ) = find_python_versions_and_site_packages(m_solution, database);
 
         // if no action required, don't even start logging them
         if (!empty())
@@ -303,7 +314,10 @@ namespace mamba
             [](specs::PackageInfo&& pkg) { return solver::Solution::Install{ std::move(pkg) }; }
         );
 
-        m_py_versions = find_python_version(m_solution, database);
+        std::tie(
+            m_py_versions,
+            m_python_site_packages_path
+        ) = find_python_versions_and_site_packages(m_solution, database);
     }
 
     class TransactionRollback
@@ -395,7 +409,12 @@ namespace mamba
         };
 
         TransactionRollback rollback;
-        TransactionContext transaction_context(ctx.transaction_params(), m_py_versions, m_requested_specs);
+        TransactionContext transaction_context(
+            ctx.transaction_params(),
+            m_py_versions,
+            m_python_site_packages_path,
+            m_requested_specs
+        );
 
         for (const specs::PackageInfo& pkg : m_solution.packages_to_remove())
         {

libmamba/src/core/transaction_context.cpp

@@ -9,12 +9,11 @@
 
 #include <reproc++/drain.hpp>
 
-#include "mamba/core/error_handling.hpp"
 #include "mamba/core/output.hpp"
 #include "mamba/util/environment.hpp"
 #include "mamba/util/string.hpp"
 
-#include "./transaction_context.hpp"
+#include "transaction_context.hpp"
 
 extern const char data_compile_pyc_py[];
 
@@ -91,17 +90,28 @@ namespace mamba
         }
     }
 
-    TransactionContext::PythonParams
-    build_python_params(std::pair<std::string, std::string> py_versions)
+    TransactionContext::PythonParams build_python_params(
+        std::pair<std::string, std::string> py_versions,
+        std::string python_site_packages_path
+    )
     {
+        if (py_versions.first.empty())
+        {
+            return {};
+        }
+
         TransactionContext::PythonParams res;
-        if (py_versions.first.size() != 0)
+        res.has_python = true;
+        res.python_version = std::move(py_versions.first);
+        res.old_python_version = std::move(py_versions.second);
+        res.short_python_version = compute_short_python_version(res.python_version);
+        res.python_path = get_python_short_path(res.short_python_version);
+        if (!python_site_packages_path.empty())
+        {
+            res.site_packages_path = python_site_packages_path;
+        }
+        else
         {
-            res.has_python = true;
-            res.python_version = std::move(py_versions.first);
-            res.old_python_version = std::move(py_versions.second);
-            res.short_python_version = compute_short_python_version(res.python_version);
-            res.python_path = get_python_short_path(res.short_python_version);
             res.site_packages_path = get_python_site_packages_short_path(res.short_python_version);
         }
         return res;
@@ -110,10 +120,13 @@ namespace mamba
     TransactionContext::TransactionContext(
         TransactionParams transaction_params,
         std::pair<std::string, std::string> py_versions,
+        std::string python_site_packages_path,
         std::vector<specs::MatchSpec> lrequested_specs
     )
         : m_transaction_params(std::move(transaction_params))
-        , m_python_params(build_python_params(std::move(py_versions)))
+        , m_python_params(
+              build_python_params(std::move(py_versions), std::move(python_site_packages_path))
+          )
         , m_requested_specs(std::move(lrequested_specs))
     {
         if (m_python_params.python_version.size() == 0)