From ebe9e91cd5659fa6941c82215cbf71a8a6f3ddad Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 7 Jun 2024 11:35:13 +0200 Subject: [PATCH 01/59] Depend on resolvo Signed-off-by: Julien Jerphanion --- libmamba/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index 5892d87577..bc52eb6878 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -432,6 +432,8 @@ find_package(yaml-cpp CONFIG REQUIRED) find_package(reproc CONFIG REQUIRED) find_package(reproc++ CONFIG REQUIRED) find_package(Libsolv MODULE REQUIRED) +find_package(Resolvo CONFIG REQUIRED) + add_subdirectory(ext/solv-cpp) macro(libmamba_create_target target_name linkage output_name) @@ -487,6 +489,7 @@ macro(libmamba_create_target target_name linkage output_name) solv::libsolv_static solv::libsolvext_static solv::cpp + Resolvo::Resolvo ) if(UNIX) @@ -633,6 +636,7 @@ macro(libmamba_create_target target_name linkage output_name) solv::libsolv solv::libsolvext solv::cpp + Resolvo::Resolvo ) # CMake 3.17 provides a LibArchive::LibArchive target that could be used instead of # LIBRARIES/INCLUDE_DIRS From 237ad2ae63f80325e01e8ed0ccae69c7caa8d6bc Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 13 Jun 2024 11:00:37 +0200 Subject: [PATCH 02/59] Add resolvo-cpp to the mamba environment Signed-off-by: Julien Jerphanion --- dev/environment-dev.yml | 1 + dev/environment-micromamba-static.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/environment-dev.yml b/dev/environment-dev.yml index c9e2afb469..cf714f0f2a 100644 --- a/dev/environment-dev.yml +++ b/dev/environment-dev.yml @@ -15,6 +15,7 @@ dependencies: - libarchive>=3.8 lgpl_* - libcurl >=7.86 - libsodium + - resolvo-cpp - libsolv >=0.7.18 - nlohmann_json - reproc-cpp >=14.2.4.post0 diff --git a/dev/environment-micromamba-static.yml b/dev/environment-micromamba-static.yml index 2dcbfdfdec..3e13582040 100644 --- a/dev/environment-micromamba-static.yml +++ b/dev/environment-micromamba-static.yml @@ -13,6 +13,7 @@ dependencies: - simdjson-static >=3.3.0 - spdlog - fmt >=11.1.0 + - resolvo-cpp - libsolv-static >=0.7.24 - yaml-cpp-static >=0.8.0 - reproc-static >=14.2.4.post0 From b8795f7d70673ef7775efe7b9a18f4bd629c05af Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 21 Jun 2024 18:15:12 +0200 Subject: [PATCH 03/59] Add a test for resolvo Many changes are required to be able to use hashmaps. Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/specs/match_spec.hpp | 1 - libmamba/tests/CMakeLists.txt | 2 + .../tests/src/solver/resolvo/test_solver.cpp | 218 ++++++++++++++++++ 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 libmamba/tests/src/solver/resolvo/test_solver.cpp diff --git a/libmamba/include/mamba/specs/match_spec.hpp b/libmamba/include/mamba/specs/match_spec.hpp index 63d5430b03..c0e27cbc7c 100644 --- a/libmamba/include/mamba/specs/match_spec.hpp +++ b/libmamba/include/mamba/specs/match_spec.hpp @@ -196,7 +196,6 @@ namespace mamba::specs return !(*this == other); } - friend struct std::hash; }; friend struct std::hash; diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index ab43650578..a6985d8977 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -104,6 +104,8 @@ set( src/core/test_transaction_context.cpp src/core/test_util.cpp src/core/test_virtual_packages.cpp + src/core/test_filesystem.cpp + src/solver/resolvo/test_solver.cpp ) message(STATUS "Building libmamba C++ tests") diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp new file mode 100644 index 0000000000..0ce1bc69fb --- /dev/null +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -0,0 +1,218 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#include + +#include +#include +#include + +#include "mamba/specs/channel.hpp" +#include "mamba/specs/package_info.hpp" + +#include "mambatests.hpp" + + +using namespace mamba; +using namespace mamba::specs; +using namespace mamba::solver; + +using namespace resolvo; + +struct PackageDatabase : public DependencyProvider { + + Pool names; + Pool strings; + + std::unordered_map solvable_ids; + std::unordered_map version_set_ids; + + // Reverse mapping + std::unordered_map solvable_map; + std::unordered_map version_set_map; + + /** + * Allocates a new requirement and return the id of the requirement. + */ + VersionSetId alloc_version_set_id( + std::string_view raw_match_spec + ) { + const MatchSpec& match_spec = MatchSpec::parse(raw_match_spec).value(); + auto got = version_set_ids.find(match_spec); + + if (got != version_set_ids.end()) { + return got->second; + } else { + auto id = VersionSetId{static_cast(version_set_ids.size())}; + version_set_ids[match_spec] = id; + version_set_map[id] = match_spec; + return id; + } + } + + SolvableId alloc_solvable( + PackageInfo solvable + ) { + auto got = solvable_ids.find(solvable); + + if (got != solvable_ids.end()) { + return got->second; + } else { + auto id = SolvableId{static_cast(solvable_ids.size())}; + solvable_ids[solvable] = id; + solvable_map[id] = solvable; + return id; + } + + } + + /** + * Returns a user-friendly string representation of the specified solvable. + * + * When formatting the solvable, it should it include both the name of + * the package and any other identifying properties. + */ + virtual String display_solvable(SolvableId solvable) { + return ""; + } + + /** + * Returns a user-friendly string representation of the name of the + * specified solvable. + */ + virtual String display_solvable_name(SolvableId solvable) { + return display_name(solvable_name(solvable)); + } + + /** + * Returns a string representation of multiple solvables merged together. + * + * When formatting the solvables, both the name of the packages and any + * other identifying properties should be included. + */ + virtual String display_merged_solvables(Slice solvable) { + return ""; + } + + /** + * Returns an object that can be used to display the given name in a + * user-friendly way. + */ + virtual String display_name(NameId name) { + return ""; + } + + /** + * Returns a user-friendly string representation of the specified version + * set. + * + * The name of the package should *not* be included in the display. Where + * appropriate, this information is added. + */ + virtual String display_version_set(VersionSetId version_set) { + return ""; + } + + /** + * Returns the string representation of the specified string. + */ + virtual String display_string(StringId string) { + return ""; + } + + /** + * Returns the name of the package that the specified version set is + * associated with. + */ + virtual NameId version_set_name(VersionSetId version_set_id) { + return NameId{}; + } + + /** + * Returns the name of the package for the given solvable. + */ + virtual NameId solvable_name(SolvableId solvable_id) { + + } + + /** + * Obtains a list of solvables that should be considered when a package + * with the given name is requested. + */ + virtual Candidates get_candidates(NameId package) { + return {}; + } + + /** + * Sort the specified solvables based on which solvable to try first. The + * solver will iteratively try to select the highest version. If a + * conflict is found with the highest version the next version is + * tried. This continues until a solution is found. + */ + virtual void sort_candidates(Slice solvables) { + + } + + /** + * Given a set of solvables, return the solvables that match the given + * version set or if `inverse` is true, the solvables that do *not* match + * the version set. + */ + virtual Vector filter_candidates(Slice candidates, + VersionSetId version_set_id, bool inverse) { + return {}; + } + + /** + * Returns the dependencies for the specified solvable. + */ + virtual Dependencies get_dependencies(SolvableId solvable) { + const PackageInfo& package_info = solvables[solvable.id]; + Dependencies dependencies; + for (auto& dep : package_info.dependencies) { + dependencies.requirements.push_back(dep); + } + + return {Vector(package.dependencies.begin(), package.dependencies.end()), Vector{package.constrains}}; + } + + +}; + +TEST_SUITE("solver::resolvo") +{ + using PackageInfo = PackageInfo; + + TEST_CASE("Simple resolution problem") { + + PackageDatabase database; + + // Create a PackageInfo for scikit-learn + PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); + + // Add the above dependencies to the PackageInfo object dependencies + scikit_learn.dependencies.push_back("joblib >=1.2.0"); + scikit_learn.dependencies.push_back("numpy >=1.19,<3"); + scikit_learn.dependencies.push_back("scipy"); + scikit_learn.dependencies.push_back("threadpoolctl >=3.1.0"); + + // Create a PackageInfo for numpy + PackageInfo numpy("numpy", "1.21.0", "py310h4a8c4bd_0", 0); + + // Create a PackageInfo for scipy + PackageInfo scipy("scipy", "1.7.0", "py310h4a8c4bd_0", 0); + scipy.dependencies.push_back("numpy >=1.19,<3"); + + // Create a PackageInfo for joblib + PackageInfo joblib("joblib", "1.2.0", "py310h4a8c4bd_0", 0); + + // Create a PackageInfo for threadpoolctl + PackageInfo threadpoolctl("threadpoolctl", "3.1.0", "py310h4a8c4bd_0", 0); + + + + } +} \ No newline at end of file From 7bab46017bdaffbfe3c0bbf7ac031e5f20be82b7 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 21 Jun 2024 20:27:22 +0200 Subject: [PATCH 04/59] Add specializations of `std::hash` Required to use `std::unordered_map` with custom types. Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 0ce1bc69fb..74d61b1052 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -22,6 +22,21 @@ using namespace mamba::solver; using namespace resolvo; +template <> +struct std::hash { + std::size_t operator()(const VersionSetId& id) const { + return std::hash{}(id.id); + } +}; + +template <> +struct std::hash { + std::size_t operator()(const SolvableId& id) const { + return std::hash{}(id.id); + } +}; + + struct PackageDatabase : public DependencyProvider { Pool names; @@ -170,25 +185,25 @@ struct PackageDatabase : public DependencyProvider { * Returns the dependencies for the specified solvable. */ virtual Dependencies get_dependencies(SolvableId solvable) { - const PackageInfo& package_info = solvables[solvable.id]; - Dependencies dependencies; - for (auto& dep : package_info.dependencies) { - dependencies.requirements.push_back(dep); - } - - return {Vector(package.dependencies.begin(), package.dependencies.end()), Vector{package.constrains}}; +// const PackageInfo& package_info = [solvable]; +// Dependencies dependencies; +// for (auto& dep : package_info.dependencies) { +// dependencies.requirements.push_back(dep); +// } +// +// return {Vector(package.dependencies.begin(), package.dependencies.end()), Vector{package.constrains}}; } }; -TEST_SUITE("solver::resolvo") +TEST_CASE("solver::resolvo") { using PackageInfo = PackageInfo; - TEST_CASE("Simple resolution problem") { + SECTION("Simple resolution problem") { - PackageDatabase database; + // PackageDatabase database; // Create a PackageInfo for scikit-learn PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); From d2a3160a744099d1994f5fd0b71320d8a3c3d300 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 24 Jun 2024 10:24:27 +0200 Subject: [PATCH 05/59] Implement most methods Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 132 ++++++++++++++---- 1 file changed, 102 insertions(+), 30 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 74d61b1052..9e5705954f 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -36,23 +36,40 @@ struct std::hash { } }; +template <> +struct std::hash { + std::size_t operator()(const NameId& id) const { + return std::hash{}(id.id); + } +}; + +template <> +struct std::hash { + std::size_t operator()(const StringId& id) const { + return std::hash{}(id.id); + } +}; struct PackageDatabase : public DependencyProvider { - Pool names; - Pool strings; + std::unordered_map name_map; + std::unordered_map name_ids; - std::unordered_map solvable_ids; - std::unordered_map version_set_ids; + std::unordered_map string_ids; + std::unordered_map string_map; - // Reverse mapping + // PackageInfo are Solvable in resolvo's semantics + std::unordered_map solvable_ids; std::unordered_map solvable_map; + + // MatchSpec are VersionSet in resolvo's semantics + std::unordered_map version_set_ids; std::unordered_map version_set_map; /** * Allocates a new requirement and return the id of the requirement. */ - VersionSetId alloc_version_set_id( + VersionSetId alloc_version_set( std::string_view raw_match_spec ) { const MatchSpec& match_spec = MatchSpec::parse(raw_match_spec).value(); @@ -69,19 +86,18 @@ struct PackageDatabase : public DependencyProvider { } SolvableId alloc_solvable( - PackageInfo solvable + PackageInfo package_info ) { - auto got = solvable_ids.find(solvable); + auto got = solvable_ids.find(package_info); if (got != solvable_ids.end()) { return got->second; } else { auto id = SolvableId{static_cast(solvable_ids.size())}; - solvable_ids[solvable] = id; - solvable_map[id] = solvable; + solvable_ids[package_info] = id; + solvable_map[id] = package_info; return id; } - } /** @@ -91,7 +107,8 @@ struct PackageDatabase : public DependencyProvider { * the package and any other identifying properties. */ virtual String display_solvable(SolvableId solvable) { - return ""; + const PackageInfo& package_info = solvable_map[solvable]; + return String{package_info.long_str()}; } /** @@ -99,7 +116,8 @@ struct PackageDatabase : public DependencyProvider { * specified solvable. */ virtual String display_solvable_name(SolvableId solvable) { - return display_name(solvable_name(solvable)); + const PackageInfo& package_info = solvable_map[solvable]; + return String{package_info.name}; } /** @@ -109,7 +127,12 @@ struct PackageDatabase : public DependencyProvider { * other identifying properties should be included. */ virtual String display_merged_solvables(Slice solvable) { - return ""; + std::string result; + for (auto& solvable_id : solvable) { + // Append "solvable_id" and its name to the result + result += std::to_string(solvable_id.id) + " " + solvable_map[solvable_id].name + "\n"; + } + return String{result}; } /** @@ -117,7 +140,7 @@ struct PackageDatabase : public DependencyProvider { * user-friendly way. */ virtual String display_name(NameId name) { - return ""; + return name_map[name]; } /** @@ -128,14 +151,15 @@ struct PackageDatabase : public DependencyProvider { * appropriate, this information is added. */ virtual String display_version_set(VersionSetId version_set) { - return ""; + const MatchSpec match_spec = version_set_map[version_set]; + return String{match_spec.str()}; } /** * Returns the string representation of the specified string. */ virtual String display_string(StringId string) { - return ""; + return string_ids[string]; } /** @@ -143,14 +167,16 @@ struct PackageDatabase : public DependencyProvider { * associated with. */ virtual NameId version_set_name(VersionSetId version_set_id) { - return NameId{}; + const MatchSpec match_spec = version_set_map[version_set_id]; + return name_ids[String{match_spec.name().str()}]; } /** * Returns the name of the package for the given solvable. */ virtual NameId solvable_name(SolvableId solvable_id) { - + const PackageInfo& package_info = solvable_map[solvable_id]; + return name_ids[String{package_info.name}]; } /** @@ -158,7 +184,14 @@ struct PackageDatabase : public DependencyProvider { * with the given name is requested. */ virtual Candidates get_candidates(NameId package) { - return {}; + Candidates candidates; + // TODO: inefficient for now, O(n) which can be turned into O(1) + for (auto& [solvable_id, package_info] : solvable_map) { + if (package == solvable_name(solvable_id)) { + candidates.candidates.push_back(solvable_id); + } + } + return candidates; } /** @@ -168,7 +201,12 @@ struct PackageDatabase : public DependencyProvider { * tried. This continues until a solution is found. */ virtual void sort_candidates(Slice solvables) { - + std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { + const PackageInfo& package_info_a = solvable_map[a]; + const PackageInfo& package_info_b = solvable_map[b]; + // TODO: Add some caching on the version parsing + return Version::parse(package_info_a.version).value() < Version::parse(package_info_b.version).value(); + }); } /** @@ -178,20 +216,54 @@ struct PackageDatabase : public DependencyProvider { */ virtual Vector filter_candidates(Slice candidates, VersionSetId version_set_id, bool inverse) { - return {}; + Vector filtered; + + if(inverse) { + for (auto& solvable_id : candidates) + { + const PackageInfo& package_info = solvable_map[solvable_id]; + const MatchSpec match_spec = version_set_map[version_set_id]; + + // Is it an appropriate check? Or must another one be crafted? + if (!match_spec.contains_except_channel(package_info)) + { + filtered.push_back(solvable_id); + } + } + } else { + for (auto& solvable_id : candidates) + { + const PackageInfo& package_info = solvable_map[solvable_id]; + const MatchSpec match_spec = version_set_map[version_set_id]; + + // Is it an appropriate check? Or must another one be crafted? + if (match_spec.contains_except_channel(package_info)) + { + filtered.push_back(solvable_id); + } + } + } + + return filtered; } /** * Returns the dependencies for the specified solvable. */ - virtual Dependencies get_dependencies(SolvableId solvable) { -// const PackageInfo& package_info = [solvable]; -// Dependencies dependencies; -// for (auto& dep : package_info.dependencies) { -// dependencies.requirements.push_back(dep); -// } -// -// return {Vector(package.dependencies.begin(), package.dependencies.end()), Vector{package.constrains}}; + virtual Dependencies get_dependencies(SolvableId solvable_id) { + const PackageInfo& package_info = solvable_map[solvable_id]; + Dependencies dependencies; + + for (auto& dep : package_info.dependencies) { + const MatchSpec match_spec = MatchSpec::parse(dep).value(); + dependencies.requirements.push_back(version_set_ids[match_spec]); + } + for (auto& constr : package_info.constrains) { + const MatchSpec match_spec = MatchSpec::parse(constr).value(); + dependencies.constrains.push_back(version_set_ids[match_spec]); + } + + return dependencies; } From 4a0b12c89be3e053413f26e3da07a397824a290e Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 27 Jun 2024 18:12:43 +0200 Subject: [PATCH 06/59] wip Signed-off-by: Julien Jerphanion --- libmamba/tests/CMakeLists.txt | 5 +- .../tests/src/solver/resolvo/test_solver.cpp | 56 ++++++++++++++++++- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index a6985d8977..b1f592821d 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -127,10 +127,7 @@ find_package(Threads REQUIRED) target_link_libraries( test_libmamba PUBLIC mamba::libmamba reproc reproc++ - PRIVATE Catch2::Catch2WithMain Threads::Threads -) -set_target_properties( - test_libmamba PROPERTIES COMPILE_DEFINITIONS CATCH_CONFIG_ENABLE_ALL_STRINGMAKERS + PRIVATE Catch2::Catch2WithMain Threads::Threads Resolvo::Resolvo ) # Copy data directory into binary dir to avoid modifications diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 9e5705954f..6b5fe4ea49 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -72,15 +73,20 @@ struct PackageDatabase : public DependencyProvider { VersionSetId alloc_version_set( std::string_view raw_match_spec ) { - const MatchSpec& match_spec = MatchSpec::parse(raw_match_spec).value(); + const MatchSpec match_spec = MatchSpec::parse(raw_match_spec).value(); auto got = version_set_ids.find(match_spec); if (got != version_set_ids.end()) { + std::cout << "Found version set id for " << raw_match_spec << std::endl; return got->second; } else { + std::cout << "Allocating version set id for " << raw_match_spec << std::endl; auto id = VersionSetId{static_cast(version_set_ids.size())}; + std::cout << "Allocated version set id " << id.id << std::endl; version_set_ids[match_spec] = id; + std::cout << "Added version set id to map" << std::endl; version_set_map[id] = match_spec; + std::cout << "Added version set to map" << std::endl; return id; } } @@ -90,12 +96,29 @@ struct PackageDatabase : public DependencyProvider { ) { auto got = solvable_ids.find(package_info); + const std::string& name = package_info.name; + if (got != solvable_ids.end()) { + std::cout << "Found solvable id for " << name << std::endl; return got->second; } else { + std::cout << "Allocating solvable id for " << name << std::endl; auto id = SolvableId{static_cast(solvable_ids.size())}; solvable_ids[package_info] = id; solvable_map[id] = package_info; + // For each of the dependencies, allocate a version set + for (auto& dep : package_info.dependencies) { + alloc_version_set(dep); + } + // Populate the names and the string + auto name_id = NameId{static_cast(name_map.size())}; + name_map[name_id] = name; + name_ids[String{name}] = name_id; + + auto string_id = StringId{static_cast(string_ids.size())}; + string_ids[string_id] = name; + string_map[String{name}] = string_id; + return id; } } @@ -168,6 +191,7 @@ struct PackageDatabase : public DependencyProvider { */ virtual NameId version_set_name(VersionSetId version_set_id) { const MatchSpec match_spec = version_set_map[version_set_id]; + std::cout << "Getting name id for version_set_id " << match_spec.name().str() << std::endl; return name_ids[String{match_spec.name().str()}]; } @@ -176,6 +200,7 @@ struct PackageDatabase : public DependencyProvider { */ virtual NameId solvable_name(SolvableId solvable_id) { const PackageInfo& package_info = solvable_map[solvable_id]; + std::cout << "Getting name id for solvable " << package_info.name << std::endl; return name_ids[String{package_info.name}]; } @@ -184,6 +209,7 @@ struct PackageDatabase : public DependencyProvider { * with the given name is requested. */ virtual Candidates get_candidates(NameId package) { + std::cout << "Getting candidates for " << name_map[package] << std::endl; Candidates candidates; // TODO: inefficient for now, O(n) which can be turned into O(1) for (auto& [solvable_id, package_info] : solvable_map) { @@ -201,6 +227,7 @@ struct PackageDatabase : public DependencyProvider { * tried. This continues until a solution is found. */ virtual void sort_candidates(Slice solvables) { + std::cout << "Sorting candidates" << std::endl; std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { const PackageInfo& package_info_a = solvable_map[a]; const PackageInfo& package_info_b = solvable_map[b]; @@ -252,6 +279,7 @@ struct PackageDatabase : public DependencyProvider { */ virtual Dependencies get_dependencies(SolvableId solvable_id) { const PackageInfo& package_info = solvable_map[solvable_id]; + std::cout << "Getting dependencies for " << package_info.name << std::endl; Dependencies dependencies; for (auto& dep : package_info.dependencies) { @@ -275,7 +303,7 @@ TEST_CASE("solver::resolvo") SECTION("Simple resolution problem") { - // PackageDatabase database; + PackageDatabase database; // Create a PackageInfo for scikit-learn PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); @@ -299,7 +327,29 @@ TEST_CASE("solver::resolvo") // Create a PackageInfo for threadpoolctl PackageInfo threadpoolctl("threadpoolctl", "3.1.0", "py310h4a8c4bd_0", 0); - + // Allocate all the PackageInfo + database.alloc_solvable(scikit_learn); + database.alloc_solvable(numpy); + database.alloc_solvable(scipy); + database.alloc_solvable(joblib); + database.alloc_solvable(threadpoolctl); + + // Construct a problem to be solved by the solver + resolvo::Vector requirements = { + database.alloc_version_set("scikit-learn>=1.5.0"), + }; + resolvo::Vector constraints = {}; + + // Solve the problem + std::cout << "Solving the problem" << std::endl; + resolvo::Vector result; + resolvo::solve(database, requirements, constraints, result); + + // Display the result + std::cout << "Result contains " << result.size() << " solvables" << std::endl; + for (auto& solvable_id : result) { + std::cout << database.display_solvable(solvable_id) << std::endl; + } } } \ No newline at end of file From baca592e5cad6eaf76825ff3491f376a39d5a411 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 28 Jun 2024 10:36:37 +0200 Subject: [PATCH 07/59] wip Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 79 ++++++++++++++----- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 6b5fe4ea49..d260b49d10 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -74,6 +74,7 @@ struct PackageDatabase : public DependencyProvider { std::string_view raw_match_spec ) { const MatchSpec match_spec = MatchSpec::parse(raw_match_spec).value(); + const std::string name = match_spec.name().str(); auto got = version_set_ids.find(match_spec); if (got != version_set_ids.end()) { @@ -87,6 +88,25 @@ struct PackageDatabase : public DependencyProvider { std::cout << "Added version set id to map" << std::endl; version_set_map[id] = match_spec; std::cout << "Added version set to map" << std::endl; + + auto got_name = name_ids.find(String{name}); + + if (got_name == name_ids.end()) { + auto name_id = NameId{static_cast(name_map.size())}; + name_map[name_id] = name; + name_ids[String{name}] = name_id; + return id; + } + + auto got_string = string_map.find(String{name}); + + if (got_string == string_map.end()) { + auto string_id = StringId{static_cast(string_ids.size())}; + string_ids[string_id] = name; + string_map[String{name}] = string_id; + return id; + } + return id; } } @@ -96,7 +116,7 @@ struct PackageDatabase : public DependencyProvider { ) { auto got = solvable_ids.find(package_info); - const std::string& name = package_info.name; + const std::string name = package_info.name; if (got != solvable_ids.end()) { std::cout << "Found solvable id for " << name << std::endl; @@ -106,18 +126,31 @@ struct PackageDatabase : public DependencyProvider { auto id = SolvableId{static_cast(solvable_ids.size())}; solvable_ids[package_info] = id; solvable_map[id] = package_info; - // For each of the dependencies, allocate a version set + for (auto& dep : package_info.dependencies) { alloc_version_set(dep); } - // Populate the names and the string - auto name_id = NameId{static_cast(name_map.size())}; - name_map[name_id] = name; - name_ids[String{name}] = name_id; + for (auto& constr : package_info.constrains) { + alloc_version_set(constr); + } + + auto got_name = name_ids.find(String{name}); + + if (got_name == name_ids.end()) { + auto name_id = NameId{static_cast(name_map.size())}; + name_map[name_id] = name; + name_ids[String{name}] = name_id; + return id; + } - auto string_id = StringId{static_cast(string_ids.size())}; - string_ids[string_id] = name; - string_map[String{name}] = string_id; + auto got_string = string_map.find(String{name}); + + if (got_string == string_map.end()) { + auto string_id = StringId{static_cast(string_ids.size())}; + string_ids[string_id] = name; + string_map[String{name}] = string_id; + return id; + } return id; } @@ -153,7 +186,7 @@ struct PackageDatabase : public DependencyProvider { std::string result; for (auto& solvable_id : solvable) { // Append "solvable_id" and its name to the result - result += std::to_string(solvable_id.id) + " " + solvable_map[solvable_id].name + "\n"; + result += std::to_string(solvable_id.id) + " " + solvable_map[solvable_id].build_string + "\n"; } return String{result}; } @@ -213,7 +246,9 @@ struct PackageDatabase : public DependencyProvider { Candidates candidates; // TODO: inefficient for now, O(n) which can be turned into O(1) for (auto& [solvable_id, package_info] : solvable_map) { + std::cout << " Checking " << package_info.name << " " << package_info.version << std::endl; if (package == solvable_name(solvable_id)) { + std::cout << " Adding candidate " << package_info.name << std::endl; candidates.candidates.push_back(solvable_id); } } @@ -306,20 +341,21 @@ TEST_CASE("solver::resolvo") PackageDatabase database; // Create a PackageInfo for scikit-learn + PackageInfo scikit_learn0("scikit-learn", "1.5.0", "py310h981052a_0", 0); PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); // Add the above dependencies to the PackageInfo object dependencies - scikit_learn.dependencies.push_back("joblib >=1.2.0"); - scikit_learn.dependencies.push_back("numpy >=1.19,<3"); - scikit_learn.dependencies.push_back("scipy"); - scikit_learn.dependencies.push_back("threadpoolctl >=3.1.0"); + scikit_learn.dependencies.push_back("joblib==1.2.0"); + // scikit_learn.dependencies.push_back("numpy >=1.19,<3"); + // scikit_learn.dependencies.push_back("scipy"); + // scikit_learn.dependencies.push_back("threadpoolctl >=3.1.0"); // Create a PackageInfo for numpy PackageInfo numpy("numpy", "1.21.0", "py310h4a8c4bd_0", 0); // Create a PackageInfo for scipy PackageInfo scipy("scipy", "1.7.0", "py310h4a8c4bd_0", 0); - scipy.dependencies.push_back("numpy >=1.19,<3"); + // scipy.dependencies.push_back("numpy >=1.19,<3"); // Create a PackageInfo for joblib PackageInfo joblib("joblib", "1.2.0", "py310h4a8c4bd_0", 0); @@ -328,22 +364,25 @@ TEST_CASE("solver::resolvo") PackageInfo threadpoolctl("threadpoolctl", "3.1.0", "py310h4a8c4bd_0", 0); // Allocate all the PackageInfo + database.alloc_solvable(scikit_learn0); database.alloc_solvable(scikit_learn); - database.alloc_solvable(numpy); - database.alloc_solvable(scipy); + // database.alloc_solvable(numpy); + // database.alloc_solvable(scipy); database.alloc_solvable(joblib); - database.alloc_solvable(threadpoolctl); + // database.alloc_solvable(threadpoolctl); // Construct a problem to be solved by the solver resolvo::Vector requirements = { - database.alloc_version_set("scikit-learn>=1.5.0"), + database.alloc_version_set("scikit-learn==1.5.0"), }; resolvo::Vector constraints = {}; // Solve the problem std::cout << "Solving the problem" << std::endl; resolvo::Vector result; - resolvo::solve(database, requirements, constraints, result); + String reason = resolvo::solve(database, requirements, constraints, result); + + std::cout << "Reason: " << reason << std::endl; // Display the result std::cout << "Result contains " << result.size() << " solvables" << std::endl; From 5d326ddd22cdc7c72d1c4734a93c79db38217a16 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 28 Jun 2024 13:41:39 +0200 Subject: [PATCH 08/59] wip: Use custom pools Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 221 +++++++++--------- 1 file changed, 111 insertions(+), 110 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index d260b49d10..8ed6334430 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -51,21 +51,64 @@ struct std::hash { } }; -struct PackageDatabase : public DependencyProvider { - std::unordered_map name_map; - std::unordered_map name_ids; +// Create a template Pool class that maps a key to a set of values +template +struct Mapping { + Mapping() = default; + ~Mapping() = default; - std::unordered_map string_ids; - std::unordered_map string_map; + /** + * Adds the value to the Mapping and returns its associated id. If the + * value is already in the Mapping, returns the id associated with it. + */ + ID alloc(T value) { + if (auto element = value_to_id.find(value); element != value_to_id.end()) { + return element->second; + } + auto id = ID{static_cast(id_to_value.size())}; + id_to_value[id] = value; + value_to_id[value] = id; + return id; + } - // PackageInfo are Solvable in resolvo's semantics - std::unordered_map solvable_ids; - std::unordered_map solvable_map; + /** + * Returns the value associated with the given id. + */ + T operator[](ID id) { return id_to_value[id]; } + + /** + * Returns the id associated with the given value. + */ + ID operator[](T value) { return value_to_id[value]; } + + // Iterator for the Mapping + auto begin() { return id_to_value.begin(); } + auto end() { return id_to_value.end(); } + auto begin() const { return id_to_value.begin(); } + auto end() const { return id_to_value.end(); } + auto cbegin() { return id_to_value.cbegin(); } + auto cend() { return id_to_value.cend(); } + auto cbegin() const { return id_to_value.cbegin(); } + auto cend() const { return id_to_value.cend(); } + auto find(T value) { return value_to_id.find(value); } + +private: + std::unordered_map value_to_id; + std::unordered_map id_to_value; +}; + + +struct PackageDatabase : public DependencyProvider { + + ::Mapping name_pool; + ::Mapping string_pool; // MatchSpec are VersionSet in resolvo's semantics - std::unordered_map version_set_ids; - std::unordered_map version_set_map; + ::Mapping version_set_pool; + + // PackageInfo are Solvable in resolvo's semantics + ::Mapping solvable_pool; /** * Allocates a new requirement and return the id of the requirement. @@ -75,85 +118,41 @@ struct PackageDatabase : public DependencyProvider { ) { const MatchSpec match_spec = MatchSpec::parse(raw_match_spec).value(); const std::string name = match_spec.name().str(); - auto got = version_set_ids.find(match_spec); - if (got != version_set_ids.end()) { - std::cout << "Found version set id for " << raw_match_spec << std::endl; - return got->second; - } else { - std::cout << "Allocating version set id for " << raw_match_spec << std::endl; - auto id = VersionSetId{static_cast(version_set_ids.size())}; - std::cout << "Allocated version set id " << id.id << std::endl; - version_set_ids[match_spec] = id; - std::cout << "Added version set id to map" << std::endl; - version_set_map[id] = match_spec; - std::cout << "Added version set to map" << std::endl; - - auto got_name = name_ids.find(String{name}); - - if (got_name == name_ids.end()) { - auto name_id = NameId{static_cast(name_map.size())}; - name_map[name_id] = name; - name_ids[String{name}] = name_id; - return id; - } + // Add the version set to the version set pool + auto id = version_set_pool.alloc(match_spec); - auto got_string = string_map.find(String{name}); + // Add name to the name pool + name_pool.alloc(String{name}); - if (got_string == string_map.end()) { - auto string_id = StringId{static_cast(string_ids.size())}; - string_ids[string_id] = name; - string_map[String{name}] = string_id; - return id; - } + // Add name to the string pool + string_pool.alloc(String{name}); - return id; - } + return id; } SolvableId alloc_solvable( PackageInfo package_info ) { - auto got = solvable_ids.find(package_info); - const std::string name = package_info.name; - if (got != solvable_ids.end()) { - std::cout << "Found solvable id for " << name << std::endl; - return got->second; - } else { - std::cout << "Allocating solvable id for " << name << std::endl; - auto id = SolvableId{static_cast(solvable_ids.size())}; - solvable_ids[package_info] = id; - solvable_map[id] = package_info; - - for (auto& dep : package_info.dependencies) { - alloc_version_set(dep); - } - for (auto& constr : package_info.constrains) { - alloc_version_set(constr); - } - - auto got_name = name_ids.find(String{name}); + // Add the solvable to the solvable pool + auto id = solvable_pool.alloc(package_info); - if (got_name == name_ids.end()) { - auto name_id = NameId{static_cast(name_map.size())}; - name_map[name_id] = name; - name_ids[String{name}] = name_id; - return id; - } + // Add name to the name pool + name_pool.alloc(String{name}); - auto got_string = string_map.find(String{name}); + // Add name to the string pool + string_pool.alloc(String{name}); - if (got_string == string_map.end()) { - auto string_id = StringId{static_cast(string_ids.size())}; - string_ids[string_id] = name; - string_map[String{name}] = string_id; - return id; - } - - return id; + for (auto& dep : package_info.dependencies) { + alloc_version_set(dep); } + for (auto& constr : package_info.constrains) { + alloc_version_set(constr); + } + + return id; } /** @@ -162,8 +161,8 @@ struct PackageDatabase : public DependencyProvider { * When formatting the solvable, it should it include both the name of * the package and any other identifying properties. */ - virtual String display_solvable(SolvableId solvable) { - const PackageInfo& package_info = solvable_map[solvable]; + String display_solvable(SolvableId solvable) override { + const PackageInfo& package_info = solvable_pool[solvable]; return String{package_info.long_str()}; } @@ -171,8 +170,8 @@ struct PackageDatabase : public DependencyProvider { * Returns a user-friendly string representation of the name of the * specified solvable. */ - virtual String display_solvable_name(SolvableId solvable) { - const PackageInfo& package_info = solvable_map[solvable]; + String display_solvable_name(SolvableId solvable) override { + const PackageInfo& package_info = solvable_pool[solvable]; return String{package_info.name}; } @@ -182,11 +181,11 @@ struct PackageDatabase : public DependencyProvider { * When formatting the solvables, both the name of the packages and any * other identifying properties should be included. */ - virtual String display_merged_solvables(Slice solvable) { + String display_merged_solvables(Slice solvable) override { std::string result; for (auto& solvable_id : solvable) { // Append "solvable_id" and its name to the result - result += std::to_string(solvable_id.id) + " " + solvable_map[solvable_id].build_string + "\n"; + result += std::to_string(solvable_id.id) + " " + solvable_pool[solvable_id].build_string + "\n"; } return String{result}; } @@ -195,8 +194,8 @@ struct PackageDatabase : public DependencyProvider { * Returns an object that can be used to display the given name in a * user-friendly way. */ - virtual String display_name(NameId name) { - return name_map[name]; + String display_name(NameId name) override { + return name_pool[name]; } /** @@ -206,46 +205,46 @@ struct PackageDatabase : public DependencyProvider { * The name of the package should *not* be included in the display. Where * appropriate, this information is added. */ - virtual String display_version_set(VersionSetId version_set) { - const MatchSpec match_spec = version_set_map[version_set]; + String display_version_set(VersionSetId version_set) override { + const MatchSpec match_spec = version_set_pool[version_set]; return String{match_spec.str()}; } /** * Returns the string representation of the specified string. */ - virtual String display_string(StringId string) { - return string_ids[string]; + String display_string(StringId string) override { + return string_pool[string]; } /** * Returns the name of the package that the specified version set is * associated with. */ - virtual NameId version_set_name(VersionSetId version_set_id) { - const MatchSpec match_spec = version_set_map[version_set_id]; + NameId version_set_name(VersionSetId version_set_id) override { + const MatchSpec match_spec = version_set_pool[version_set_id]; std::cout << "Getting name id for version_set_id " << match_spec.name().str() << std::endl; - return name_ids[String{match_spec.name().str()}]; + return name_pool[String{match_spec.name().str()}]; } /** * Returns the name of the package for the given solvable. */ - virtual NameId solvable_name(SolvableId solvable_id) { - const PackageInfo& package_info = solvable_map[solvable_id]; + NameId solvable_name(SolvableId solvable_id) override { + const PackageInfo& package_info = solvable_pool[solvable_id]; std::cout << "Getting name id for solvable " << package_info.name << std::endl; - return name_ids[String{package_info.name}]; + return name_pool[String{package_info.name}]; } /** * Obtains a list of solvables that should be considered when a package * with the given name is requested. */ - virtual Candidates get_candidates(NameId package) { - std::cout << "Getting candidates for " << name_map[package] << std::endl; + Candidates get_candidates(NameId package) override { + std::cout << "Getting candidates for " << name_pool[package] << std::endl; Candidates candidates; // TODO: inefficient for now, O(n) which can be turned into O(1) - for (auto& [solvable_id, package_info] : solvable_map) { + for (auto& [solvable_id, package_info] : solvable_pool) { std::cout << " Checking " << package_info.name << " " << package_info.version << std::endl; if (package == solvable_name(solvable_id)) { std::cout << " Adding candidate " << package_info.name << std::endl; @@ -261,11 +260,11 @@ struct PackageDatabase : public DependencyProvider { * conflict is found with the highest version the next version is * tried. This continues until a solution is found. */ - virtual void sort_candidates(Slice solvables) { + void sort_candidates(Slice solvables) override { std::cout << "Sorting candidates" << std::endl; std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { - const PackageInfo& package_info_a = solvable_map[a]; - const PackageInfo& package_info_b = solvable_map[b]; + const PackageInfo& package_info_a = solvable_pool[a]; + const PackageInfo& package_info_b = solvable_pool[b]; // TODO: Add some caching on the version parsing return Version::parse(package_info_a.version).value() < Version::parse(package_info_b.version).value(); }); @@ -276,15 +275,18 @@ struct PackageDatabase : public DependencyProvider { * version set or if `inverse` is true, the solvables that do *not* match * the version set. */ - virtual Vector filter_candidates(Slice candidates, - VersionSetId version_set_id, bool inverse) { + Vector filter_candidates( + Slice candidates, + VersionSetId version_set_id, + bool inverse + ) override { Vector filtered; if(inverse) { for (auto& solvable_id : candidates) { - const PackageInfo& package_info = solvable_map[solvable_id]; - const MatchSpec match_spec = version_set_map[version_set_id]; + const PackageInfo& package_info = solvable_pool[solvable_id]; + const MatchSpec match_spec = version_set_pool[version_set_id]; // Is it an appropriate check? Or must another one be crafted? if (!match_spec.contains_except_channel(package_info)) @@ -295,8 +297,8 @@ struct PackageDatabase : public DependencyProvider { } else { for (auto& solvable_id : candidates) { - const PackageInfo& package_info = solvable_map[solvable_id]; - const MatchSpec match_spec = version_set_map[version_set_id]; + const PackageInfo& package_info = solvable_pool[solvable_id]; + const MatchSpec match_spec = version_set_pool[version_set_id]; // Is it an appropriate check? Or must another one be crafted? if (match_spec.contains_except_channel(package_info)) @@ -312,24 +314,23 @@ struct PackageDatabase : public DependencyProvider { /** * Returns the dependencies for the specified solvable. */ - virtual Dependencies get_dependencies(SolvableId solvable_id) { - const PackageInfo& package_info = solvable_map[solvable_id]; + Dependencies get_dependencies(SolvableId solvable_id) override { + const PackageInfo& package_info = solvable_pool[solvable_id]; std::cout << "Getting dependencies for " << package_info.name << std::endl; Dependencies dependencies; for (auto& dep : package_info.dependencies) { const MatchSpec match_spec = MatchSpec::parse(dep).value(); - dependencies.requirements.push_back(version_set_ids[match_spec]); + dependencies.requirements.push_back(version_set_pool[match_spec]); } for (auto& constr : package_info.constrains) { const MatchSpec match_spec = MatchSpec::parse(constr).value(); - dependencies.constrains.push_back(version_set_ids[match_spec]); + dependencies.constrains.push_back(version_set_pool[match_spec]); } return dependencies; } - }; TEST_CASE("solver::resolvo") @@ -345,7 +346,7 @@ TEST_CASE("solver::resolvo") PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); // Add the above dependencies to the PackageInfo object dependencies - scikit_learn.dependencies.push_back("joblib==1.2.0"); + scikit_learn.dependencies.emplace_back("joblib==1.2.0"); // scikit_learn.dependencies.push_back("numpy >=1.19,<3"); // scikit_learn.dependencies.push_back("scipy"); // scikit_learn.dependencies.push_back("threadpoolctl >=3.1.0"); From 183e365b738912761e661caedcd6cfd5dea8e372 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 28 Jun 2024 16:56:16 +0200 Subject: [PATCH 09/59] Minimal Reproducer of the problem Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 40 +++++-------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 8ed6334430..90fa0323d2 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -101,6 +101,8 @@ struct Mapping { struct PackageDatabase : public DependencyProvider { + virtual ~PackageDatabase() = default; + ::Mapping name_pool; ::Mapping string_pool; @@ -185,7 +187,8 @@ struct PackageDatabase : public DependencyProvider { std::string result; for (auto& solvable_id : solvable) { // Append "solvable_id" and its name to the result - result += std::to_string(solvable_id.id) + " " + solvable_pool[solvable_id].build_string + "\n"; + std::cout << "Displaying solvable " << solvable_id.id << " " << solvable_pool[solvable_id].long_str() << std::endl; + result += std::to_string(solvable_id.id) + " " + solvable_pool[solvable_id].long_str(); } return String{result}; } @@ -341,36 +344,13 @@ TEST_CASE("solver::resolvo") PackageDatabase database; - // Create a PackageInfo for scikit-learn - PackageInfo scikit_learn0("scikit-learn", "1.5.0", "py310h981052a_0", 0); - PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_1", 1); - - // Add the above dependencies to the PackageInfo object dependencies - scikit_learn.dependencies.emplace_back("joblib==1.2.0"); - // scikit_learn.dependencies.push_back("numpy >=1.19,<3"); - // scikit_learn.dependencies.push_back("scipy"); - // scikit_learn.dependencies.push_back("threadpoolctl >=3.1.0"); - - // Create a PackageInfo for numpy - PackageInfo numpy("numpy", "1.21.0", "py310h4a8c4bd_0", 0); - - // Create a PackageInfo for scipy - PackageInfo scipy("scipy", "1.7.0", "py310h4a8c4bd_0", 0); - // scipy.dependencies.push_back("numpy >=1.19,<3"); - - // Create a PackageInfo for joblib - PackageInfo joblib("joblib", "1.2.0", "py310h4a8c4bd_0", 0); - - // Create a PackageInfo for threadpoolctl - PackageInfo threadpoolctl("threadpoolctl", "3.1.0", "py310h4a8c4bd_0", 0); - - // Allocate all the PackageInfo - database.alloc_solvable(scikit_learn0); + // NOTE: the problem can only be solved when two `Solvalble` are added to the `PackageDatabase` + PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); database.alloc_solvable(scikit_learn); - // database.alloc_solvable(numpy); - // database.alloc_solvable(scipy); - database.alloc_solvable(joblib); - // database.alloc_solvable(threadpoolctl); + + PackageInfo scikit_learn_bis("scikit-learn", "1.5.0", "py310h981052a_1", 1); + // NOTE: Uncomment this line and the problem becomes solvable. + // database.alloc_solvable(scikit_learn_bis); // Construct a problem to be solved by the solver resolvo::Vector requirements = { From c28affb34924c65e5da5c25b848617012512b1b9 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 11:06:00 +0200 Subject: [PATCH 10/59] Also sort on build number Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 90fa0323d2..06720ad5e8 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -269,7 +269,14 @@ struct PackageDatabase : public DependencyProvider { const PackageInfo& package_info_a = solvable_pool[a]; const PackageInfo& package_info_b = solvable_pool[b]; // TODO: Add some caching on the version parsing - return Version::parse(package_info_a.version).value() < Version::parse(package_info_b.version).value(); + const auto a_version = Version::parse(package_info_a.version).value(); + const auto b_version = Version::parse(package_info_b.version).value(); + + if (a_version != b_version) { + return a_version < b_version; + } + + return package_info_a.build_number < package_info_b.build_number; }); } @@ -340,11 +347,41 @@ TEST_CASE("solver::resolvo") { using PackageInfo = PackageInfo; + SECTION("Sort solvables increasing order") { + PackageDatabase database; + + PackageInfo skl0("scikit-learn", "1.5.2", "py310h981052a_0", 0); + auto sol0 = database.alloc_solvable(skl0); + + PackageInfo skl1("scikit-learn", "1.5.0", "py310h981052a_1", 1); + auto sol1 = database.alloc_solvable(skl1); + + PackageInfo skl2("scikit-learn", "1.5.1", "py310h981052a_2", 2); + auto sol2 = database.alloc_solvable(skl2); + + PackageInfo skl3("scikit-learn", "1.5.0", "py310h981052a_2", 2); + auto sol3 = database.alloc_solvable(skl3); + + PackageInfo scikit_learn_ter("scikit-learn", "1.5.1", "py310h981052a_1", 1); + auto sol4 = database.alloc_solvable(skl3); + + Vector solvables = {sol0, sol1, sol2, sol3, sol4}; + + database.sort_candidates(solvables); + + CHECK(solvables[0] == sol1); + CHECK(solvables[1] == sol3); + CHECK(solvables[2] == sol4); + CHECK(solvables[3] == sol2); + CHECK(solvables[4] == sol0); + + } + SECTION("Simple resolution problem") { PackageDatabase database; - // NOTE: the problem can only be solved when two `Solvalble` are added to the `PackageDatabase` + // NOTE: the problem can only be solved when two `Solvable` are added to the `PackageDatabase` PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); database.alloc_solvable(scikit_learn); From de9d09b1b879225381462ce43728d6b33598a554 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 11:52:50 +0200 Subject: [PATCH 11/59] test: Addition of PackageInfo to PackageDatabase Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 06720ad5e8..5129a427c8 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -93,6 +93,16 @@ struct Mapping { auto cend() const { return id_to_value.cend(); } auto find(T value) { return value_to_id.find(value); } + auto begin_ids() { return value_to_id.begin(); } + auto end_ids() { return value_to_id.end(); } + auto begin_ids() const { return value_to_id.begin(); } + auto end_ids() const { return value_to_id.end(); } + auto cbegin_ids() { return value_to_id.cbegin(); } + auto cend_ids() { return value_to_id.cend(); } + auto cbegin_ids() const { return value_to_id.cbegin(); } + auto cend_ids() const { return value_to_id.cend(); } + + private: std::unordered_map value_to_id; std::unordered_map id_to_value; @@ -347,6 +357,46 @@ TEST_CASE("solver::resolvo") { using PackageInfo = PackageInfo; + SECTION("Addition of PackageInfo to PackageDatabase") { + + PackageDatabase database; + + PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); + scikit_learn.dependencies.emplace_back("numpy >=1.20.0,<2.0a0"); + scikit_learn.dependencies.emplace_back("scipy >=1.6.0,<2.0a0"); + scikit_learn.dependencies.emplace_back("joblib >=1.0.1,<2.0a0"); + scikit_learn.dependencies.emplace_back("threadpoolctl >=2.1.0,<3.0a0"); + + auto solvable = database.alloc_solvable(scikit_learn); + + CHECK(solvable.id == 0); + CHECK(database.solvable_pool[solvable].name == "scikit-learn"); + CHECK(database.solvable_pool[solvable].version == "1.5.0"); + CHECK(database.solvable_pool[solvable].build_string == "py310h981052a_0"); + CHECK(database.solvable_pool[solvable].build_number == 0); + + auto deps = database.get_dependencies(solvable); + CHECK(deps.requirements.size() == 4); + CHECK(deps.constrains.size() == 0); + + CHECK(database.version_set_pool[deps.requirements[0]].str() == "numpy[version=\">=1.20.0,<2.0a0\"]"); + CHECK(database.version_set_pool[deps.requirements[1]].str() == "scipy[version=\">=1.6.0,<2.0a0\"]"); + CHECK(database.version_set_pool[deps.requirements[2]].str() == "joblib[version=\">=1.0.1,<2.0a0\"]"); + CHECK(database.version_set_pool[deps.requirements[3]].str() == "threadpoolctl[version=\">=2.1.0,<3.0a0\"]"); + + CHECK(database.name_pool.find(String{"scikit-learn"}) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{"numpy"}) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{"scipy"}) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{"joblib"}) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{"threadpoolctl"}) != database.name_pool.end_ids()); + + CHECK(database.string_pool.find(String{"scikit-learn"}) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{"numpy"}) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{"scipy"}) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{"joblib"}) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{"threadpoolctl"}) != database.string_pool.end_ids()); + } + SECTION("Sort solvables increasing order") { PackageDatabase database; From 0fc96769a433a4b571bf9403c017bd8c531136cc Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 12:29:20 +0200 Subject: [PATCH 12/59] test: Filter solvables Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 5129a427c8..0fd77d46dc 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -397,6 +397,70 @@ TEST_CASE("solver::resolvo") CHECK(database.string_pool.find(String{"threadpoolctl"}) != database.string_pool.end_ids()); } + SECTION("Filter solvables") { + PackageDatabase database; + + PackageInfo skl0("scikit-learn", "1.4.0", "py310h981052a_0", 0); + auto sol0 = database.alloc_solvable(skl0); + + PackageInfo skl1("scikit-learn", "1.5.0", "py310h981052a_1", 1); + auto sol1 = database.alloc_solvable(skl1); + + PackageInfo skl2("scikit-learn", "1.5.1", "py310h981052a_0", 0); + auto sol2 = database.alloc_solvable(skl2); + + PackageInfo skl3("scikit-learn", "1.5.1", "py310h981052a_2", 2); + auto sol3 = database.alloc_solvable(skl3); + + auto solvables = Vector{sol0, sol1, sol2, sol3}; + + // Filter on scikit-learn + auto all = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn"), false); + CHECK(all.size() == 4); + CHECK(all[0] == sol0); + CHECK(all[1] == sol1); + CHECK(all[2] == sol2); + CHECK(all[3] == sol3); + + // Inverse filter on scikit-learn + auto none = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn"), true); + CHECK(none.size() == 0); + + // Filter on scikit-learn==1.5.1 + auto one = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn==1.5.1"), false); + CHECK(one.size() == 2); + CHECK(one[0] == sol2); + CHECK(one[1] == sol3); + + // Inverse filter on scikit-learn==1.5.1 + auto three = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn==1.5.1"), true); + CHECK(three.size() == 2); + CHECK(three[0] == sol0); + CHECK(three[1] == sol1); + + // Filter on scikit-learn<1.5.1 + auto two = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn<1.5.1"), false); + CHECK(two.size() == 2); + CHECK(two[0] == sol0); + CHECK(two[1] == sol1); + + // Filter on build number 0 + auto build = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==0]"), false); + CHECK(build.size() == 2); + CHECK(build[0] == sol0); + CHECK(build[1] == sol2); + + // Filter on build number 2 + auto build_bis = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==2]"), false); + CHECK(build_bis.size() == 1); + CHECK(build_bis[0] == sol3); + + // Filter on build number 3 + auto build_ter = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==3]"), false); + CHECK(build_ter.size() == 0); + + } + SECTION("Sort solvables increasing order") { PackageDatabase database; @@ -459,4 +523,5 @@ TEST_CASE("solver::resolvo") } } + } \ No newline at end of file From 7f05c6fcef85bee791c9d4e7e54c1054209924a2 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 13:06:03 +0200 Subject: [PATCH 13/59] Add more strings and names to pools Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 0fd77d46dc..897a1839d0 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -129,34 +129,39 @@ struct PackageDatabase : public DependencyProvider { std::string_view raw_match_spec ) { const MatchSpec match_spec = MatchSpec::parse(raw_match_spec).value(); - const std::string name = match_spec.name().str(); // Add the version set to the version set pool auto id = version_set_pool.alloc(match_spec); - // Add name to the name pool + // Add name to the Name and String pools + const std::string name = match_spec.name().str(); name_pool.alloc(String{name}); - - // Add name to the string pool string_pool.alloc(String{name}); + // Add the MatchSpec's string representation to the Name and String pools + const std::string match_spec_str = match_spec.str(); + name_pool.alloc(String{match_spec_str}); + string_pool.alloc(String{match_spec_str}); + return id; } SolvableId alloc_solvable( PackageInfo package_info ) { - const std::string name = package_info.name; - // Add the solvable to the solvable pool auto id = solvable_pool.alloc(package_info); - // Add name to the name pool + // Add name to the Name and String pools + const std::string name = package_info.name; name_pool.alloc(String{name}); - - // Add name to the string pool string_pool.alloc(String{name}); + // Add the long string representation of the package to the Name and String pools + const std::string long_str = package_info.long_str(); + name_pool.alloc(String{long_str}); + string_pool.alloc(String{long_str}); + for (auto& dep : package_info.dependencies) { alloc_version_set(dep); } From e97546d684e03a829738e7d0612b058a98c95ffb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 15:57:31 +0200 Subject: [PATCH 14/59] Initialize Candidates.{favored,locked} to nullptr Resolve the problem with the locking. Signed-off-by: Julien Jerphanion Co-authored-by: Wolf Vollprecht --- .../tests/src/solver/resolvo/test_solver.cpp | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 897a1839d0..e785d9f8c6 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -261,6 +261,8 @@ struct PackageDatabase : public DependencyProvider { Candidates get_candidates(NameId package) override { std::cout << "Getting candidates for " << name_pool[package] << std::endl; Candidates candidates; + candidates.favored = nullptr; + candidates.locked = nullptr; // TODO: inefficient for now, O(n) which can be turned into O(1) for (auto& [solvable_id, package_info] : solvable_pool) { std::cout << " Checking " << package_info.name << " " << package_info.version << std::endl; @@ -496,37 +498,24 @@ TEST_CASE("solver::resolvo") } - SECTION("Simple resolution problem") { + SECTION("Trivial problem") { PackageDatabase database; - // NOTE: the problem can only be solved when two `Solvable` are added to the `PackageDatabase` PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); database.alloc_solvable(scikit_learn); - PackageInfo scikit_learn_bis("scikit-learn", "1.5.0", "py310h981052a_1", 1); - // NOTE: Uncomment this line and the problem becomes solvable. - // database.alloc_solvable(scikit_learn_bis); - - // Construct a problem to be solved by the solver resolvo::Vector requirements = { database.alloc_version_set("scikit-learn==1.5.0"), }; resolvo::Vector constraints = {}; - // Solve the problem - std::cout << "Solving the problem" << std::endl; resolvo::Vector result; String reason = resolvo::solve(database, requirements, constraints, result); - std::cout << "Reason: " << reason << std::endl; - - // Display the result - std::cout << "Result contains " << result.size() << " solvables" << std::endl; - for (auto& solvable_id : result) { - std::cout << database.display_solvable(solvable_id) << std::endl; - } - + CHECK(reason == ""); + CHECK(result.size() == 1); + CHECK(database.solvable_pool[result[0]] == scikit_learn); } } \ No newline at end of file From 6dbe87c0721c7ab1ce7017a76078d4833ce03585 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 15:08:26 +0200 Subject: [PATCH 15/59] wip Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index e785d9f8c6..25c7db2b0d 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -250,7 +250,7 @@ struct PackageDatabase : public DependencyProvider { */ NameId solvable_name(SolvableId solvable_id) override { const PackageInfo& package_info = solvable_pool[solvable_id]; - std::cout << "Getting name id for solvable " << package_info.name << std::endl; + std::cout << "Getting name id for solvable " << package_info.long_str() << std::endl; return name_pool[String{package_info.name}]; } @@ -265,9 +265,9 @@ struct PackageDatabase : public DependencyProvider { candidates.locked = nullptr; // TODO: inefficient for now, O(n) which can be turned into O(1) for (auto& [solvable_id, package_info] : solvable_pool) { - std::cout << " Checking " << package_info.name << " " << package_info.version << std::endl; + std::cout << " Checking " << package_info.long_str() << std::endl; if (package == solvable_name(solvable_id)) { - std::cout << " Adding candidate " << package_info.name << std::endl; + std::cout << " Adding candidate " << package_info.long_str() << std::endl; candidates.candidates.push_back(solvable_id); } } @@ -292,6 +292,7 @@ struct PackageDatabase : public DependencyProvider { if (a_version != b_version) { return a_version < b_version; } + // TODO: add sorting on track features and other things return package_info_a.build_number < package_info_b.build_number; }); @@ -343,7 +344,7 @@ struct PackageDatabase : public DependencyProvider { */ Dependencies get_dependencies(SolvableId solvable_id) override { const PackageInfo& package_info = solvable_pool[solvable_id]; - std::cout << "Getting dependencies for " << package_info.name << std::endl; + std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; Dependencies dependencies; for (auto& dep : package_info.dependencies) { @@ -518,4 +519,91 @@ TEST_CASE("solver::resolvo") CHECK(database.solvable_pool[result[0]] == scikit_learn); } + TEST_CASE("Register repodata.json") + { + PackageDatabase database; + + + /* + * { + "info": {"subdir":"linux-64"}, + "packages": { + "21cmfast-3.0.2-py36h1af98f8_1.tar.bz2": { + "build":"py36h1af98f8_1", + "build_number":1, + "constrains":[ + "llvm-meta ==8.0.0", + "llvmdev ==8.0.0" + ], + "depends": [ + "_openmp_mutex >=4.5", + "astropy >=2.0", + "cached-property", + "cffi >=1.0", + "click", + "fftw >=3.3.8,<4.0a0", + "gsl >=2.6,<2.7.0a0", + "h5py >=2.8.0", + "libblas >=3.8.0,<4.0a0", + "libgcc-ng >=7.5.0", + "matplotlib-base", + "numpy", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "pyyaml", + "scipy" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "d65ab674acf3b7294ebacaec05fc5b54", + "name": "21cmfast", + "sha256": "1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2", + "size": 414494, + "subdir": "linux-64", + "timestamp": 1605110689658, + "version": "3.0.2" + } + }, + "removed": [], + "repodata_version": 1 + } + */ + // Create a PackageInfo from the above JSON + PackageInfo package_info; + package_info.name = "21cmfast"; + package_info.version = "3.0.2"; + package_info.build_string = "py36h1af98f8_1"; + package_info.build_number = 1; + package_info.channel = "conda-forge/linux-64"; + package_info.filename = "21cmfast-3.0.2-py36h1af98f8_1.tar.bz2"; + package_info.license = "MIT"; + package_info.md5 = "d65ab674acf3b7294ebacaec05fc5b54"; + package_info.sha256 = "1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2"; + package_info.size = 414494; + package_info.timestamp = 1605110689658; + package_info.package_type = PackageType::Conda; + + package_info.build_number = 1; + package_info.constrains = {"llvm-meta ==8.0.0", "llvmdev ==8.0.0"}; + package_info.dependencies = { + "_openmp_mutex >=4.5", + "astropy >=2.0", + "cached-property", + "cffi >=1.0", + "click", + "fftw >=3.3.8,<4.0a0", + "gsl >=2.6,<2.7.0a0", + "h5py >=2.8.0", + "libblas >=3.8.0,<4.0a0", + "libgcc-ng >=7.5.0", + "matplotlib-base", + "numpy", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "pyyaml", + "scipy" + }; + } + + } \ No newline at end of file From b5a7dcd2f5ef4bfbf1be967c74afec38b30d219a Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 1 Jul 2024 18:46:03 +0200 Subject: [PATCH 16/59] Parse repodata.json Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/specs/match_spec.hpp | 1 + libmamba/tests/CMakeLists.txt | 6 +- .../tests/src/solver/resolvo/test_solver.cpp | 548 +++++++++++++++--- 3 files changed, 461 insertions(+), 94 deletions(-) diff --git a/libmamba/include/mamba/specs/match_spec.hpp b/libmamba/include/mamba/specs/match_spec.hpp index c0e27cbc7c..63d5430b03 100644 --- a/libmamba/include/mamba/specs/match_spec.hpp +++ b/libmamba/include/mamba/specs/match_spec.hpp @@ -196,6 +196,7 @@ namespace mamba::specs return !(*this == other); } + friend struct std::hash; }; friend struct std::hash; diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index b1f592821d..c52f0d3f4e 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -104,7 +104,6 @@ set( src/core/test_transaction_context.cpp src/core/test_util.cpp src/core/test_virtual_packages.cpp - src/core/test_filesystem.cpp src/solver/resolvo/test_solver.cpp ) @@ -127,7 +126,10 @@ find_package(Threads REQUIRED) target_link_libraries( test_libmamba PUBLIC mamba::libmamba reproc reproc++ - PRIVATE Catch2::Catch2WithMain Threads::Threads Resolvo::Resolvo + PRIVATE Catch2::Catch2WithMain Threads::Threads Resolvo::Resolvo simdjson::simdjson +) +set_target_properties( + test_libmamba PROPERTIES COMPILE_DEFINITIONS CATCH_CONFIG_ENABLE_ALL_STRINGMAKERS ) # Copy data directory into binary dir to avoid modifications diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 25c7db2b0d..2ae1973d1e 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -11,9 +11,15 @@ #include #include +#include + +#include "mamba/core/util.hpp" // for LockFile #include "mamba/specs/channel.hpp" #include "mamba/specs/package_info.hpp" +// TODO: move PackageTypes and MAX_CONDA_TIMESTAMP to a common place +#include "mamba/solver/libsolv/parameters.hpp" // for PackageTypes + #include "mambatests.hpp" @@ -102,6 +108,8 @@ struct Mapping { auto cbegin_ids() const { return value_to_id.cbegin(); } auto cend_ids() const { return value_to_id.cend(); } + auto size() const { return id_to_value.size(); } + private: std::unordered_map value_to_id; @@ -128,8 +136,82 @@ struct PackageDatabase : public DependencyProvider { VersionSetId alloc_version_set( std::string_view raw_match_spec ) { - const MatchSpec match_spec = MatchSpec::parse(raw_match_spec).value(); + std::string raw_match_spec_str = std::string(raw_match_spec); + // Replace all " v" with simply " " to work around the `v` prefix in some version strings + // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in `infom2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` + while (raw_match_spec_str.find(" v") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(" v"), 2, " "); + } + + // Remove any presence of selector on python version in the match spec + // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in `infowillow-1.6.3-pyhd8ed1ab_0.conda` + for(const auto specifier: {"=py", "py", ">=py", "<=py", "!=py"}) + { + while (raw_match_spec_str.find(specifier) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(specifier)); + } + } + // Remove any white space between version + // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` + while (raw_match_spec_str.find(", ") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(", "), 2, ","); + } + + // TODO: skip allocation for now if "*.*" is in the match spec + if (raw_match_spec_str.find("*.*") != std::string::npos) + { + return VersionSetId{0}; + } + + // NOTE: works around `openblas 0.2.18|0.2.18.*.` from `dlib==19.0=np110py27_blas_openblas_200` + // If contains "|", split on it and recurse + if (raw_match_spec_str.find("|") != std::string::npos) + { + std::vector match_specs; + std::string match_spec; + for (char c : raw_match_spec_str) + { + if (c == '|') + { + match_specs.push_back(match_spec); + match_spec.clear(); + } + else + { + match_spec += c; + } + } + match_specs.push_back(match_spec); + std::vector version_sets; + for (const std::string& match_spec : match_specs) + { + alloc_version_set(match_spec); + } + // Placeholder return value + return VersionSetId{0}; + } + + // NOTE: This works around some improperly encoded `constrains` in the test data, e.g.: + // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit >= 10.2" + // `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" + // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: ">=4.5.2" + // Remove any with space after the binary operators + for(const std::string& op : {">=", "<=", "==", ">", "<", "!=", "=", "=="}) { + const std::string& bad_op = op + " "; + while (raw_match_spec_str.find(bad_op) != std::string::npos) { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op + raw_match_spec_str.substr(raw_match_spec_str.find(bad_op) + bad_op.size()); + } + // If start with binary operator, prepend NONE + if (raw_match_spec_str.find(op) == 0) { + raw_match_spec_str = "NONE " + raw_match_spec_str; + } + } + + const MatchSpec match_spec = MatchSpec::parse(raw_match_spec_str).value(); // Add the version set to the version set pool auto id = version_set_pool.alloc(match_spec); @@ -142,7 +224,6 @@ struct PackageDatabase : public DependencyProvider { const std::string match_spec_str = match_spec.str(); name_pool.alloc(String{match_spec_str}); string_pool.alloc(String{match_spec_str}); - return id; } @@ -202,7 +283,7 @@ struct PackageDatabase : public DependencyProvider { std::string result; for (auto& solvable_id : solvable) { // Append "solvable_id" and its name to the result - std::cout << "Displaying solvable " << solvable_id.id << " " << solvable_pool[solvable_id].long_str() << std::endl; + // std::cout << "Displaying solvable " << solvable_id.id << " " << solvable_pool[solvable_id].long_str() << std::endl; result += std::to_string(solvable_id.id) + " " + solvable_pool[solvable_id].long_str(); } return String{result}; @@ -241,7 +322,7 @@ struct PackageDatabase : public DependencyProvider { */ NameId version_set_name(VersionSetId version_set_id) override { const MatchSpec match_spec = version_set_pool[version_set_id]; - std::cout << "Getting name id for version_set_id " << match_spec.name().str() << std::endl; + // std::cout << "Getting name id for version_set_id " << match_spec.name().str() << std::endl; return name_pool[String{match_spec.name().str()}]; } @@ -250,7 +331,7 @@ struct PackageDatabase : public DependencyProvider { */ NameId solvable_name(SolvableId solvable_id) override { const PackageInfo& package_info = solvable_pool[solvable_id]; - std::cout << "Getting name id for solvable " << package_info.long_str() << std::endl; + // std::cout << "Getting name id for solvable " << package_info.long_str() << std::endl; return name_pool[String{package_info.name}]; } @@ -259,15 +340,15 @@ struct PackageDatabase : public DependencyProvider { * with the given name is requested. */ Candidates get_candidates(NameId package) override { - std::cout << "Getting candidates for " << name_pool[package] << std::endl; + // std::cout << "Getting candidates for " << name_pool[package] << std::endl; Candidates candidates; candidates.favored = nullptr; candidates.locked = nullptr; // TODO: inefficient for now, O(n) which can be turned into O(1) for (auto& [solvable_id, package_info] : solvable_pool) { - std::cout << " Checking " << package_info.long_str() << std::endl; + // std::cout << " Checking " << package_info.long_str() << std::endl; if (package == solvable_name(solvable_id)) { - std::cout << " Adding candidate " << package_info.long_str() << std::endl; + // std::cout << " Adding candidate " << package_info.long_str() << std::endl; candidates.candidates.push_back(solvable_id); } } @@ -281,7 +362,7 @@ struct PackageDatabase : public DependencyProvider { * tried. This continues until a solution is found. */ void sort_candidates(Slice solvables) override { - std::cout << "Sorting candidates" << std::endl; + // std::cout << "Sorting candidates" << std::endl; std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { const PackageInfo& package_info_a = solvable_pool[a]; const PackageInfo& package_info_b = solvable_pool[b]; @@ -290,11 +371,11 @@ struct PackageDatabase : public DependencyProvider { const auto b_version = Version::parse(package_info_b.version).value(); if (a_version != b_version) { - return a_version < b_version; + return a_version > b_version; } // TODO: add sorting on track features and other things - return package_info_a.build_number < package_info_b.build_number; + return package_info_a.build_number > package_info_b.build_number; }); } @@ -344,7 +425,7 @@ struct PackageDatabase : public DependencyProvider { */ Dependencies get_dependencies(SolvableId solvable_id) override { const PackageInfo& package_info = solvable_pool[solvable_id]; - std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; + // std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; Dependencies dependencies; for (auto& dep : package_info.dependencies) { @@ -361,6 +442,241 @@ struct PackageDatabase : public DependencyProvider { }; +// TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` +auto lsplit_track_features(std::string_view features) +{ + constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); +} + +bool parse_packageinfo_json( + const std::string_view& filename, + const simdjson::dom::element& pkg, + PackageDatabase& database + ) { + PackageInfo package_info; + + package_info.filename = filename; + if (auto fn = pkg["fn"].get_string(); !fn.error()) + { + package_info.name = fn.value_unsafe(); + } + else + { + // Fallback from key entry + package_info.name = filename; + } + + if (auto name = pkg["name"].get_string(); !name.error()) + { + package_info.name = name.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")"; + return false; + } + + if (auto version = pkg["version"].get_string(); !version.error()) + { + package_info.version = version.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")"; + return false; + } + + if (auto build_string = pkg["build"].get_string(); !build_string.error()) + { + package_info.build_string = build_string.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")"; + return false; + } + + if (auto build_number = pkg["build_number"].get_uint64(); !build_number.error()) + { + package_info.build_number = build_number.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")"; + return false; + } + + if (auto subdir = pkg["subdir"].get_c_str(); !subdir.error()) + { + package_info.platform = subdir.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid subdir in ")" << filename << R"(")"; + } + + if (auto size = pkg["size"].get_uint64(); !size.error()) + { + package_info.size = size.value_unsafe(); + } + + if (auto md5 = pkg["md5"].get_c_str(); !md5.error()) + { + package_info.md5 = md5.value_unsafe(); + } + + if (auto sha256 = pkg["sha256"].get_c_str(); !sha256.error()) + { + package_info.sha256 = sha256.value_unsafe(); + } + + if (auto elem = pkg["noarch"]; !elem.error()) + { + // TODO: is the following right? + if (auto val = elem.get_bool(); !val.error() && val.value_unsafe()) + { + package_info.noarch = NoArchType::No; + } + else if (auto noarch = elem.get_c_str(); !noarch.error()) + { + package_info.noarch = NoArchType::No; + } + } + + if (auto license = pkg["license"].get_c_str(); !license.error()) + { + package_info.license = license.value_unsafe(); + } + + // TODO conda timestamp are not Unix timestamp. + // Libsolv normalize them this way, we need to do the same here otherwise the current + // package may get arbitrary priority. + if (auto timestamp = pkg["timestamp"].get_uint64(); !timestamp.error()) + { + const auto time = timestamp.value_unsafe(); + // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` + constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; + package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; + } + + if (auto depends = pkg["depends"].get_array(); !depends.error()) + { + for (auto elem : depends) + { + if (auto dep = elem.get_c_str(); !dep.error()) + { + package_info.dependencies.emplace_back(dep.value_unsafe()); + } + } + } + + if (auto constrains = pkg["constrains"].get_array(); !constrains.error()) + { + for (auto elem : constrains) + { + if (auto cons = elem.get_c_str(); !cons.error()) + { + package_info.constrains.emplace_back(cons.value_unsafe()); + } + } + } + + if (auto obj = pkg["track_features"]; !obj.error()) + { + if (auto track_features_arr = obj.get_array(); !track_features_arr.error()) + { + for (auto elem : track_features_arr) + { + if (auto feat = elem.get_string(); !feat.error()) + { + package_info.track_features.emplace_back(feat.value_unsafe()); + } + } + } + else if (auto track_features_str = obj.get_string(); !track_features_str.error()) + { + auto splits = lsplit_track_features(track_features_str.value_unsafe()); + while (!splits[0].empty()) + { + package_info.track_features.emplace_back(splits[0]); + splits = lsplit_track_features(splits[1]); + } + } + } + + database.alloc_solvable(package_info); + return true; +} + +void parse_repodata_json( + PackageDatabase& database, + const fs::u8path& filename, + const std::string& repo_url, + const std::string& channel_id, + bool verify_artifacts +) +{ + auto parser = simdjson::dom::parser(); + const auto lock = LockFile(filename); + const auto repodata = parser.load(filename); + + // An override for missing package subdir is found at the top level + auto default_subdir = std::string(); + if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); !subdir.error()) + { + default_subdir = std::string(subdir.value_unsafe()); + } + + // Get `base_url` in case 'repodata_version': 2 + // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md + auto base_url = repo_url; + if (auto repodata_version = repodata["repodata_version"].get_int64(); + !repodata_version.error()) + { + if (repodata_version.value_unsafe() == 2) + { + if (auto url = repodata.at_pointer("/info/base_url").get_string(); !url.error()) + { + base_url = std::string(url.value_unsafe()); + } + } + } + + const auto parsed_url = specs::CondaURL::parse(base_url) + .or_else([](specs::ParseError&& err) { throw std::move(err); }) + .value(); + + auto signatures = std::optional(std::nullopt); + if (auto maybe_sigs = repodata["signatures"].get_object(); + !maybe_sigs.error() && verify_artifacts) + { + signatures = std::move(maybe_sigs).value(); + } + + auto added = util::flat_set(); + if (auto pkgs = repodata["packages.conda"].get_object(); !pkgs.error()) + { + std::cout << "CondaOrElseTarBz2 packages.conda" << std::endl; + + for (auto [key, value] : pkgs.value()) + { + parse_packageinfo_json(key, value, database); + } + } + if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) + { + std::cout << "CondaOrElseTarBz2 packages" << std::endl; + + for (auto [key, value] : pkgs.value()) + { + parse_packageinfo_json(key, value, database); + } + } + +} + TEST_CASE("solver::resolvo") { using PackageInfo = PackageInfo; @@ -519,91 +835,139 @@ TEST_CASE("solver::resolvo") CHECK(database.solvable_pool[result[0]] == scikit_learn); } - TEST_CASE("Register repodata.json") + SECTION("Parse linux-64/repodata.json") { PackageDatabase database; + parse_repodata_json( + database, + "/tmp/linux-64/repodata.json", + "https://conda.anaconda.org/conda-forge/linux-64/repodata.json", + "conda-forge", + false + ); + + std::cout << "Number of solvables: " << database.solvable_pool.size() << std::endl; - /* - * { - "info": {"subdir":"linux-64"}, - "packages": { - "21cmfast-3.0.2-py36h1af98f8_1.tar.bz2": { - "build":"py36h1af98f8_1", - "build_number":1, - "constrains":[ - "llvm-meta ==8.0.0", - "llvmdev ==8.0.0" - ], - "depends": [ - "_openmp_mutex >=4.5", - "astropy >=2.0", - "cached-property", - "cffi >=1.0", - "click", - "fftw >=3.3.8,<4.0a0", - "gsl >=2.6,<2.7.0a0", - "h5py >=2.8.0", - "libblas >=3.8.0,<4.0a0", - "libgcc-ng >=7.5.0", - "matplotlib-base", - "numpy", - "python >=3.6,<3.7.0a0", - "python_abi 3.6.* *_cp36m", - "pyyaml", - "scipy" - ], - "license": "MIT", - "license_family": "MIT", - "md5": "d65ab674acf3b7294ebacaec05fc5b54", - "name": "21cmfast", - "sha256": "1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2", - "size": 414494, - "subdir": "linux-64", - "timestamp": 1605110689658, - "version": "3.0.2" - } - }, - "removed": [], - "repodata_version": 1 - } - */ - // Create a PackageInfo from the above JSON - PackageInfo package_info; - package_info.name = "21cmfast"; - package_info.version = "3.0.2"; - package_info.build_string = "py36h1af98f8_1"; - package_info.build_number = 1; - package_info.channel = "conda-forge/linux-64"; - package_info.filename = "21cmfast-3.0.2-py36h1af98f8_1.tar.bz2"; - package_info.license = "MIT"; - package_info.md5 = "d65ab674acf3b7294ebacaec05fc5b54"; - package_info.sha256 = "1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2"; - package_info.size = 414494; - package_info.timestamp = 1605110689658; - package_info.package_type = PackageType::Conda; - - package_info.build_number = 1; - package_info.constrains = {"llvm-meta ==8.0.0", "llvmdev ==8.0.0"}; - package_info.dependencies = { - "_openmp_mutex >=4.5", - "astropy >=2.0", - "cached-property", - "cffi >=1.0", - "click", - "fftw >=3.3.8,<4.0a0", - "gsl >=2.6,<2.7.0a0", - "h5py >=2.8.0", - "libblas >=3.8.0,<4.0a0", - "libgcc-ng >=7.5.0", - "matplotlib-base", - "numpy", - "python >=3.6,<3.7.0a0", - "python_abi 3.6.* *_cp36m", - "pyyaml", - "scipy" - }; } + SECTION("Parse noarch/repodata.json") + { + PackageDatabase database; + + parse_repodata_json( + database, + "/tmp/noarch/repodata.json", + "https://conda.anaconda.org/conda-forge/noarch/repodata.json", + "conda-forge", + false + ); + + std::cout << "Number of solvables: " << database.solvable_pool.size() << std::endl; + + } + + SECTION("Known problem resolution") { + PackageDatabase database; + + parse_repodata_json( + database, + "/tmp/linux-64/repodata.json", + "https://conda.anaconda.org/conda-forge/linux-64/repodata.json", + "conda-forge", + false + ); + + + parse_repodata_json( + database, + "/tmp/noarch/repodata.json", + "https://conda.anaconda.org/conda-forge/noarch/repodata.json", + "conda-forge", + false + ); + + std::cout << "Solving problem" << std::endl; + + resolvo::Vector requirements = { + database.alloc_version_set("python[version=\">=3.10,<3.11.0a0\"]"), + database.alloc_version_set("pip"), + database.alloc_version_set("scikit-learn[version=\">=1.0.0,<1.6a0\"]"), + database.alloc_version_set("numpy[version=\">=1.20.0,<2.0a0\"]"), + database.alloc_version_set("scipy[version=\">=1.10.0,<1.15a0\"]"), + database.alloc_version_set("joblib[version=\">=1.0.1,<2.0a0\"]"), + database.alloc_version_set("threadpoolctl[version=\">=2.1.0,<3.6a0\"]"), + }; + resolvo::Vector constraints = {}; + + resolvo::Vector result; + + String reason = resolvo::solve(database, requirements, constraints, result); + + CHECK(reason == ""); + + CHECK(result.size() == 36); + // Sort all the `Solvables` in `result` on their name + std::sort(result.begin(), result.end(), [&](const SolvableId& a, const SolvableId& b) { + const PackageInfo& package_info_a = database.solvable_pool[a]; + const PackageInfo& package_info_b = database.solvable_pool[b]; + return package_info_a.name < package_info_b.name; + }); + + std::vector known_resolution = { + PackageInfo("_libgcc_mutex", "0.1", "conda_forge", 0), + PackageInfo("python_abi", "3.10", "4_cp310", 0), + PackageInfo("ld_impl_linux-64", "2.40", "hf3520f5_7", 0), + PackageInfo("ca-certificates", "2024.6.2", "hbcca054_0", 0), + PackageInfo("libgomp", "14.1.0", "h77fa898_0", 0), + PackageInfo("_openmp_mutex", "4.5", "2_gnu", 0), + PackageInfo("libgcc-ng", "14.1.0", "h77fa898_0", 0), + PackageInfo("openssl", "3.3.1", "h4ab18f5_1", 0), + PackageInfo("libxcrypt", "4.4.36", "hd590300_1", 0), + PackageInfo("libzlib", "1.3.1", "h4ab18f5_1", 0), + PackageInfo("libffi", "3.4.2", "h7f98852_5", 0), + PackageInfo("bzip2", "1.0.8", "hd590300_5", 0), + PackageInfo("ncurses", "6.5", "h59595ed_0", 0), + PackageInfo("libstdcxx-ng", "14.1.0", "hc0a3c3a_0", 0), + PackageInfo("libgfortran5", "14.1.0", "hc5f4f2c_0", 0), + PackageInfo("libuuid", "2.38.1", "h0b41bf4_0", 0), + PackageInfo("libnsl", "2.0.1", "hd590300_0", 0), + PackageInfo("xz", "5.2.6", "h166bdaf_0", 0), + PackageInfo("tk", "8.6.13", "noxft_h4845f30_101", 0), + PackageInfo("libsqlite", "3.46.0", "hde9e2c9_0", 0), + PackageInfo("readline", "8.2", "h8228510_1", 0), + PackageInfo("libgfortran-ng", "14.1.0", "h69a702a_0", 0), + PackageInfo("libopenblas", "0.3.27", "pthreads_h413a1c8_0", 0), + PackageInfo("libblas", "3.9.0", "22_linux64_openblas", 0), + PackageInfo("libcblas", "3.9.0", "22_linux64_openblas", 0), + PackageInfo("liblapack", "3.9.0", "22_linux64_openblas", 0), + PackageInfo("tzdata", "2024a", "h0c530f3_0", 0), + PackageInfo("python", "3.10.14", "hd12c33a_0_cpython", 0), + PackageInfo("wheel", "0.43.0", "pyhd8ed1ab_1", 0), + PackageInfo("setuptools", "70.1.1", "pyhd8ed1ab_0", 0), + PackageInfo("pip", "24.0", "pyhd8ed1ab_0", 0), + PackageInfo("threadpoolctl", "3.5.0", "pyhc1e730c_0", 0), + PackageInfo("joblib", "1.4.2", "pyhd8ed1ab_0", 0), + PackageInfo("numpy", "1.26.4", "py310hb13e2d6_0", 0), + PackageInfo("scipy", "1.14.0", "py310h93e2701_0", 0), + PackageInfo("scikit-learn", "1.5.0", "py310h981052a_1", 1) + }; + + // Sort know_resolution on their name + std::sort(known_resolution.begin(), known_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { + return a.name < b.name; + }); + + // Check solvables against know_resolution + for (size_t i = 0; i < result.size(); i++) { + const PackageInfo& package_info = database.solvable_pool[result[i]]; + const PackageInfo& known_package_info = known_resolution[i]; + // TODO: also check all the other fields + CHECK(package_info.name == known_package_info.name); + CHECK(package_info.version == known_package_info.version); + CHECK(package_info.build_string == known_package_info.build_string); + } + + } } \ No newline at end of file From 4ff86b7f299466c4404d4040f5cea0eb2618f24b Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 2 Jul 2024 11:32:51 +0200 Subject: [PATCH 17/59] wip: Test consistency with libsolv Signed-off-by: Julien Jerphanion --- .../tests/src/solver/libsolv/test_solver.cpp | 40 ++ .../tests/src/solver/resolvo/test_solver.cpp | 361 +++++++++++++++--- 2 files changed, 346 insertions(+), 55 deletions(-) diff --git a/libmamba/tests/src/solver/libsolv/test_solver.cpp b/libmamba/tests/src/solver/libsolv/test_solver.cpp index 08023fb0fe..8d40905c94 100644 --- a/libmamba/tests/src/solver/libsolv/test_solver.cpp +++ b/libmamba/tests/src/solver/libsolv/test_solver.cpp @@ -63,6 +63,46 @@ find_actions_with_name(const Solution& solution, std::string_view name) return out; } +auto find_actions(const Solution& solution) -> std::vector +{ + auto out = std::vector(); + for (const auto& action : solution.actions) + { + std::visit( + [&](const auto& act) + { + using Act = std::decay_t; + if constexpr (Solution::has_install_v) + { + out.push_back(act); + } + }, + action + ); + } + return out; +} + +auto extract_package_to_install(const Solution& solution) -> std::vector +{ + auto out = std::vector(); + for (const auto& action : find_actions(solution)) + { + std::visit( + [&](const auto& act) + { + using Act = std::decay_t; + if constexpr (Solution::has_install_v) + { + out.push_back(act.install); + } + }, + action + ); + } + return out; +} + namespace { using namespace specs::match_spec_literals; diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 2ae1973d1e..2a59fbafa9 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -13,12 +13,17 @@ #include +#include "mamba/api/install.hpp" // for parsing YAML specs + #include "mamba/core/util.hpp" // for LockFile #include "mamba/specs/channel.hpp" #include "mamba/specs/package_info.hpp" // TODO: move PackageTypes and MAX_CONDA_TIMESTAMP to a common place -#include "mamba/solver/libsolv/parameters.hpp" // for PackageTypes +#include "mamba/core/virtual_packages.hpp" +#include "mamba/solver/libsolv/database.hpp" +#include "mamba/solver/libsolv/parameters.hpp" // for PackageTypes +#include "mamba/solver/libsolv/solver.hpp" #include "mambatests.hpp" @@ -130,6 +135,9 @@ struct PackageDatabase : public DependencyProvider { // PackageInfo are Solvable in resolvo's semantics ::Mapping solvable_pool; + // PackageName to Vector + std::unordered_map> name_to_solvable; + /** * Allocates a new requirement and return the id of the requirement. */ @@ -187,9 +195,9 @@ struct PackageDatabase : public DependencyProvider { } match_specs.push_back(match_spec); std::vector version_sets; - for (const std::string& match_spec : match_specs) + for (const std::string& ms : match_specs) { - alloc_version_set(match_spec); + alloc_version_set(ms); } // Placeholder return value return VersionSetId{0}; @@ -250,6 +258,10 @@ struct PackageDatabase : public DependencyProvider { alloc_version_set(constr); } + // Add the solvable to the name_to_solvable map + const NameId name_id = name_pool.alloc(String{package_info.name}); + name_to_solvable[name_id].push_back(id); + return id; } @@ -344,14 +356,12 @@ struct PackageDatabase : public DependencyProvider { Candidates candidates; candidates.favored = nullptr; candidates.locked = nullptr; - // TODO: inefficient for now, O(n) which can be turned into O(1) - for (auto& [solvable_id, package_info] : solvable_pool) { - // std::cout << " Checking " << package_info.long_str() << std::endl; - if (package == solvable_name(solvable_id)) { - // std::cout << " Adding candidate " << package_info.long_str() << std::endl; - candidates.candidates.push_back(solvable_id); - } + + // TODO: implement copy/move-constructor + for (auto& solvable_id : name_to_solvable[package]) { + candidates.candidates.push_back(solvable_id); } + return candidates; } @@ -450,14 +460,20 @@ auto lsplit_track_features(std::string_view features) return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); } +// TODO: factorise with the implementation from `set_solvable` in `mamba/solver/libsolv/helpers.cpp` bool parse_packageinfo_json( const std::string_view& filename, const simdjson::dom::element& pkg, + const CondaURL& repo_url, + const std::string& channel_id, PackageDatabase& database ) { PackageInfo package_info; + package_info.channel = channel_id; package_info.filename = filename; + package_info.package_url = (repo_url / filename).str(specs::CondaURL::Credentials::Show); + if (auto fn = pkg["fn"].get_string(); !fn.error()) { package_info.name = fn.value_unsafe(); @@ -662,7 +678,7 @@ void parse_repodata_json( for (auto [key, value] : pkgs.value()) { - parse_packageinfo_json(key, value, database); + parse_packageinfo_json(key, value, parsed_url, channel_id, database); } } if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) @@ -671,14 +687,73 @@ void parse_repodata_json( for (auto [key, value] : pkgs.value()) { - parse_packageinfo_json(key, value, database); + parse_packageinfo_json(key, value, parsed_url, channel_id, database); } } +} + +// from `src/test_solver.cpp` +auto find_actions_with_name(const Solution& solution, std::string_view name) -> std::vector; +auto find_actions(const Solution& solution) -> std::vector; +auto extract_package_to_install(const Solution& solution) -> std::vector; + + +// wget https://conda.anaconda.org/conda-forge/linux-64/repodata.json +// wget https://conda.anaconda.org/conda-forge/noarch/repodata.json +auto libsolv_db = mamba::solver::libsolv::Database({ + /* .platforms= */ { "linux-64", "noarch" }, + /* .channel_alias= */ specs::CondaURL::parse("https://conda.anaconda.org/").value(), +}); + +const auto repo_linux = libsolv_db.add_repo_from_repodata_json( + "/tmp/linux-64/repodata.json", + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No +); + +const auto repo_noarch = libsolv_db.add_repo_from_repodata_json( + "/tmp/noarch/repodata.json", + "https://conda.anaconda.org/conda-forge/noarch", + "conda-forge", + libsolv::PipAsPythonDependency::Yes +); + +PackageDatabase create_resolvo_db() { + PackageDatabase resolvo_db; + + parse_repodata_json( + resolvo_db, + "/tmp/linux-64/repodata.json", + "https://conda.anaconda.org/conda-forge/linux-64/repodata.json", + "conda-forge", + false + ); + + parse_repodata_json( + resolvo_db, + "/tmp/noarch/repodata.json", + "https://conda.anaconda.org/conda-forge/noarch/repodata.json", + "conda-forge", + false + ); + + for (const auto& package : get_virtual_packages("linux-64")) + { + resolvo_db.alloc_solvable(package); + } + return resolvo_db; } + +PackageDatabase resolvo_db = create_resolvo_db(); + + TEST_CASE("solver::resolvo") { + using namespace specs::match_spec_literals; + using PackageInfo = PackageInfo; SECTION("Addition of PackageInfo to PackageDatabase") { @@ -866,53 +941,60 @@ TEST_CASE("solver::resolvo") std::cout << "Number of solvables: " << database.solvable_pool.size() << std::endl; } +} - SECTION("Known problem resolution") { - PackageDatabase database; +TEST_CASE("Test consistency with libsolv (environment creation)") { + using namespace specs::match_spec_literals; - parse_repodata_json( - database, - "/tmp/linux-64/repodata.json", - "https://conda.anaconda.org/conda-forge/linux-64/repodata.json", - "conda-forge", - false - ); + using PackageInfo = PackageInfo; + SECTION("numpy") + { + const auto request = Request{ + /* .flags= */ {}, + /* .jobs= */ { Request::Install{ "numpy"_ms } }, + }; + const auto outcome = libsolv::Solver().solve(libsolv_db, request); - parse_repodata_json( - database, - "/tmp/noarch/repodata.json", - "https://conda.anaconda.org/conda-forge/noarch/repodata.json", - "conda-forge", - false - ); + REQUIRE(outcome.has_value()); + REQUIRE(std::holds_alternative(outcome.value())); + const auto& solution = std::get(outcome.value()); + + REQUIRE_FALSE(solution.actions.empty()); - std::cout << "Solving problem" << std::endl; + // Numpy is last because of topological sort + CHECK(std::holds_alternative(solution.actions.back())); + REQUIRE(std::get(solution.actions.back()).install.name == "numpy"); + REQUIRE(find_actions_with_name(solution, "numpy").size() == 1); + + const auto python_actions = find_actions_with_name(solution, "python"); + REQUIRE(python_actions.size() == 1); + CHECK(std::holds_alternative(python_actions.front())); resolvo::Vector requirements = { - database.alloc_version_set("python[version=\">=3.10,<3.11.0a0\"]"), - database.alloc_version_set("pip"), - database.alloc_version_set("scikit-learn[version=\">=1.0.0,<1.6a0\"]"), - database.alloc_version_set("numpy[version=\">=1.20.0,<2.0a0\"]"), - database.alloc_version_set("scipy[version=\">=1.10.0,<1.15a0\"]"), - database.alloc_version_set("joblib[version=\">=1.0.1,<2.0a0\"]"), - database.alloc_version_set("threadpoolctl[version=\">=2.1.0,<3.6a0\"]"), + resolvo_db.alloc_version_set("numpy"), }; - resolvo::Vector constraints = {}; + resolvo::Vector constraints = {}; resolvo::Vector result; + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); - String reason = resolvo::solve(database, requirements, constraints, result); + REQUIRE(reason == ""); + REQUIRE(result.size() == 29); + REQUIRE(resolvo_db.solvable_pool[result[0]].name == "numpy"); + } - CHECK(reason == ""); + SECTION("scikit-learn explicit") { - CHECK(result.size() == 36); - // Sort all the `Solvables` in `result` on their name - std::sort(result.begin(), result.end(), [&](const SolvableId& a, const SolvableId& b) { - const PackageInfo& package_info_a = database.solvable_pool[a]; - const PackageInfo& package_info_b = database.solvable_pool[b]; - return package_info_a.name < package_info_b.name; - }); + std::vector specs_to_install = { + "python[version=\">=3.10,<3.11.0a0\"]", + "pip", + "scikit-learn[version=\">=1.0.0,<1.6a0\"]", + "numpy[version=\">=1.20.0,<2.0a0\"]", + "scipy[version=\">=1.10.0,<1.15a0\"]", + "joblib[version=\">=1.0.1,<2.0a0\"]", + "threadpoolctl[version=\">=2.1.0,<3.6a0\"]", + }; std::vector known_resolution = { PackageInfo("_libgcc_mutex", "0.1", "conda_forge", 0), @@ -953,21 +1035,190 @@ TEST_CASE("solver::resolvo") PackageInfo("scikit-learn", "1.5.0", "py310h981052a_1", 1) }; - // Sort know_resolution on their name std::sort(known_resolution.begin(), known_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; }); - // Check solvables against know_resolution - for (size_t i = 0; i < result.size(); i++) { - const PackageInfo& package_info = database.solvable_pool[result[i]]; + // libsolv's specification and resolution + + Request::job_list jobs; + + std::transform( + specs_to_install.begin(), + specs_to_install.end(), + std::back_inserter(jobs), + [](const std::string& spec) { + return Request::Install{MatchSpec::parse(spec).value()}; + } + ); + + const auto request = Request{ + /* .flags= */ {}, + /* .jobs= */ jobs, + }; + + const auto outcome = libsolv::Solver().solve(libsolv_db, request); + + REQUIRE(outcome.has_value()); + REQUIRE(std::holds_alternative(outcome.value())); + const auto& solution = std::get(outcome.value()); + + REQUIRE(solution.actions.size() == known_resolution.size()); + + std::vector libsolv_resolution = extract_package_to_install(solution); + std::sort(libsolv_resolution.begin(), libsolv_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { + return a.name < b.name; + }); + + // resolvo's specification and resolution + resolvo::Vector requirements; + for (const auto& spec : specs_to_install) { + requirements.push_back(resolvo_db.alloc_version_set(spec)); + } + + resolvo::Vector constraints = {}; + resolvo::Vector result; + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); + + REQUIRE(reason == ""); + REQUIRE(result.size() == known_resolution.size()); + + std::vector resolvo_resolution; + std::transform( + result.begin(), + result.end(), + std::back_inserter(resolvo_resolution), + [&](const resolvo::SolvableId& solvable_id) { + return resolvo_db.solvable_pool[solvable_id]; + } + ); + + std::sort(resolvo_resolution.begin(), resolvo_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { + return a.name < b.name; + }); + + // Check libsolv's PackageInfo against the know resolution + for (size_t i = 0; i < libsolv_resolution.size(); i++) { + const PackageInfo& package_info = libsolv_resolution[i]; const PackageInfo& known_package_info = known_resolution[i]; - // TODO: also check all the other fields - CHECK(package_info.name == known_package_info.name); - CHECK(package_info.version == known_package_info.version); - CHECK(package_info.build_string == known_package_info.build_string); + REQUIRE(package_info.name == known_package_info.name); + REQUIRE(package_info.version == known_package_info.version); + REQUIRE(package_info.build_string == known_package_info.build_string); } + // Check resolvo's PackageInfo against the know resolution + for (size_t i = 0; i < resolvo_resolution.size(); i++) { + const PackageInfo& package_info = resolvo_resolution[i]; + const PackageInfo& known_package_info = known_resolution[i]; + REQUIRE(package_info.name == known_package_info.name); + REQUIRE(package_info.version == known_package_info.version); + REQUIRE(package_info.build_string == known_package_info.build_string); + } } -} \ No newline at end of file + + SECTION("Using YAML environment specification") + { + const fs::u8path& yaml_file = "/tmp/small_spec.yaml"; // "/tmp/spec.yaml" + std::cout << "Resolving " << yaml_file << std::endl; + mamba::detail::yaml_file_contents res = mamba::detail::read_yaml_file(yaml_file, "linux-64"); + + std::vector specs_to_install = res.dependencies; + + // resolvo's specification and resolution + resolvo::Vector requirements; + for (const auto& spec : specs_to_install) + { + requirements.push_back(resolvo_db.alloc_version_set(spec)); + } + + resolvo::Vector constraints = {}; + resolvo::Vector result; + + std::cout << "Start with resolvo" << std::endl; + auto tick_resolvo = std::chrono::steady_clock::now(); + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); + auto tack_resolvo = std::chrono::steady_clock::now(); + std::cout << "End with resolvo" << std::endl; + std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; + + REQUIRE(reason == ""); + + std::vector resolvo_resolution; + std::transform( + result.begin(), + result.end(), + std::back_inserter(resolvo_resolution), + [&](const resolvo::SolvableId& solvable_id) + { return resolvo_db.solvable_pool[solvable_id]; } + ); + + std::sort( + resolvo_resolution.begin(), + resolvo_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + + // libsolv's specification and resolution + + Request::job_list jobs; + + std::transform( + specs_to_install.begin(), + specs_to_install.end(), + std::back_inserter(jobs), + [](const std::string& spec) + { return Request::Install{ MatchSpec::parse(spec).value() }; } + ); + + const auto request = Request{ + /* .flags= */ {}, + /* .jobs= */ jobs, + }; + + std::cout << "Start with libsolv" << std::endl; + auto tick_libsolv = std::chrono::steady_clock::now(); + const auto outcome = libsolv::Solver().solve(libsolv_db, request); + auto tack_libsolv = std::chrono::steady_clock::now(); + std::cout << "End with libsolv" << std::endl; + std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() << "ms" << std::endl; + + REQUIRE(outcome.has_value()); + REQUIRE(std::holds_alternative(outcome.value())); + const auto& solution = std::get(outcome.value()); + + std::vector libsolv_resolution = extract_package_to_install(solution); + std::sort( + libsolv_resolution.begin(), + libsolv_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + + // Print all the packages from libsolv + std::cout << "libsolv resolution" << std::endl; + for (const auto& package_info : libsolv_resolution) + { + std::cout << package_info.long_str() << std::endl; + } + + // Print all the packages from resolvo + std::cout << "resolvo resolution" << std::endl; + for (const auto& package_info : resolvo_resolution) + { + std::cout << package_info.long_str() << std::endl; + } + + // Check libsolv's PackageInfo against libsolv's + REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); + for (size_t i = 0; i < libsolv_resolution.size(); i++) + { + const PackageInfo& resolvo_package_info = resolvo_resolution[i]; + const PackageInfo& libsolv_package_info = libsolv_resolution[i]; + // Currently something in the parsing of the repodata.json must be different. + // TODO: find the difference and use `PackageInfo::operator==` instead + REQUIRE(resolvo_package_info.name == libsolv_package_info.name); + REQUIRE(resolvo_package_info.version == libsolv_package_info.version); + REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); + } + } +} From 5f577a108aaca908ba0695a5e3fad67dc08c0390 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 3 Jul 2024 09:03:09 +0200 Subject: [PATCH 18/59] wip Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 186 ++++++++++++++---- 1 file changed, 150 insertions(+), 36 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 2a59fbafa9..00bb8ee45b 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -294,9 +294,7 @@ struct PackageDatabase : public DependencyProvider { String display_merged_solvables(Slice solvable) override { std::string result; for (auto& solvable_id : solvable) { - // Append "solvable_id" and its name to the result - // std::cout << "Displaying solvable " << solvable_id.id << " " << solvable_pool[solvable_id].long_str() << std::endl; - result += std::to_string(solvable_id.id) + " " + solvable_pool[solvable_id].long_str(); + result += solvable_pool[solvable_id].long_str(); } return String{result}; } @@ -435,7 +433,7 @@ struct PackageDatabase : public DependencyProvider { */ Dependencies get_dependencies(SolvableId solvable_id) override { const PackageInfo& package_info = solvable_pool[solvable_id]; - // std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; + Dependencies dependencies; for (auto& dep : package_info.dependencies) { @@ -749,6 +747,87 @@ PackageDatabase create_resolvo_db() { PackageDatabase resolvo_db = create_resolvo_db(); +std::vector resolvo_resolve( + PackageDatabase& database, + const std::vector& specs +) { + // resolvo's specification and resolution + resolvo::Vector requirements; + for (const auto& spec : specs) + { + requirements.push_back(resolvo_db.alloc_version_set(spec)); + } + + resolvo::Vector constraints = {}; + resolvo::Vector result; + + std::cout << "Start with resolvo" << std::endl; + auto tick_resolvo = std::chrono::steady_clock::now(); + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); + auto tack_resolvo = std::chrono::steady_clock::now(); + std::cout << "End with resolvo" << std::endl; + std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; + + std::cout << "Resolvo's Reason: " << reason << std::endl; + + std::vector resolvo_resolution; + std::transform( + result.begin(), + result.end(), + std::back_inserter(resolvo_resolution), + [&](const resolvo::SolvableId& solvable_id) + { return resolvo_db.solvable_pool[solvable_id]; } + ); + + std::sort( + resolvo_resolution.begin(), + resolvo_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + return resolvo_resolution; +} + +std::vector libsolv_resolve( + mamba::solver::libsolv::Database& db, + const std::vector& specs +) { + // libsolv's specification and resolution + + Request::job_list jobs; + + std::transform( + specs.begin(), + specs.end(), + std::back_inserter(jobs), + [](const std::string& spec) + { return Request::Install{ MatchSpec::parse(spec).value() }; } + ); + + const auto request = Request{ + /* .flags= */ {}, + /* .jobs= */ jobs, + }; + + std::cout << "Start with libsolv" << std::endl; + auto tick_libsolv = std::chrono::steady_clock::now(); + const auto outcome = libsolv::Solver().solve(libsolv_db, request); + auto tack_libsolv = std::chrono::steady_clock::now(); + std::cout << "End with libsolv" << std::endl; + std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() << "ms" << std::endl; + + REQUIRE(outcome.has_value()); + REQUIRE(std::holds_alternative(outcome.value())); + const auto& solution = std::get(outcome.value()); + + std::vector libsolv_resolution = extract_package_to_install(solution); + std::sort( + libsolv_resolution.begin(), + libsolv_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + return libsolv_resolution; +} + TEST_CASE("solver::resolvo") { @@ -757,7 +836,6 @@ TEST_CASE("solver::resolvo") using PackageInfo = PackageInfo; SECTION("Addition of PackageInfo to PackageDatabase") { - PackageDatabase database; PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); @@ -778,22 +856,31 @@ TEST_CASE("solver::resolvo") CHECK(deps.requirements.size() == 4); CHECK(deps.constrains.size() == 0); - CHECK(database.version_set_pool[deps.requirements[0]].str() == "numpy[version=\">=1.20.0,<2.0a0\"]"); - CHECK(database.version_set_pool[deps.requirements[1]].str() == "scipy[version=\">=1.6.0,<2.0a0\"]"); - CHECK(database.version_set_pool[deps.requirements[2]].str() == "joblib[version=\">=1.0.1,<2.0a0\"]"); - CHECK(database.version_set_pool[deps.requirements[3]].str() == "threadpoolctl[version=\">=2.1.0,<3.0a0\"]"); - - CHECK(database.name_pool.find(String{"scikit-learn"}) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{"numpy"}) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{"scipy"}) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{"joblib"}) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{"threadpoolctl"}) != database.name_pool.end_ids()); - - CHECK(database.string_pool.find(String{"scikit-learn"}) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{"numpy"}) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{"scipy"}) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{"joblib"}) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{"threadpoolctl"}) != database.string_pool.end_ids()); + CHECK( + database.version_set_pool[deps.requirements[0]].str() == "numpy[version=\">=1.20.0,<2.0a0\"]" + ); + CHECK( + database.version_set_pool[deps.requirements[1]].str() == "scipy[version=\">=1.6.0,<2.0a0\"]" + ); + CHECK( + database.version_set_pool[deps.requirements[2]].str() == "joblib[version=\">=1.0.1,<2.0a0\"]" + ); + CHECK( + database.version_set_pool[deps.requirements[3]].str() + == "threadpoolctl[version=\">=2.1.0,<3.0a0\"]" + ); + + CHECK(database.name_pool.find(String{ "scikit-learn" }) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{ "numpy" }) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{ "scipy" }) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{ "joblib" }) != database.name_pool.end_ids()); + CHECK(database.name_pool.find(String{ "threadpoolctl" }) != database.name_pool.end_ids()); + + CHECK(database.string_pool.find(String{ "scikit-learn" }) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{ "numpy" }) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{ "scipy" }) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{ "joblib" }) != database.string_pool.end_ids()); + CHECK(database.string_pool.find(String{ "threadpoolctl" }) != database.string_pool.end_ids()); } SECTION("Filter solvables") { @@ -811,10 +898,14 @@ TEST_CASE("solver::resolvo") PackageInfo skl3("scikit-learn", "1.5.1", "py310h981052a_2", 2); auto sol3 = database.alloc_solvable(skl3); - auto solvables = Vector{sol0, sol1, sol2, sol3}; + auto solvables = Vector{ sol0, sol1, sol2, sol3 }; // Filter on scikit-learn - auto all = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn"), false); + auto all = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn"), + false + ); CHECK(all.size() == 4); CHECK(all[0] == sol0); CHECK(all[1] == sol1); @@ -822,42 +913,69 @@ TEST_CASE("solver::resolvo") CHECK(all[3] == sol3); // Inverse filter on scikit-learn - auto none = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn"), true); + auto none = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn"), + true + ); CHECK(none.size() == 0); // Filter on scikit-learn==1.5.1 - auto one = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn==1.5.1"), false); + auto one = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn==1.5.1"), + false + ); CHECK(one.size() == 2); CHECK(one[0] == sol2); CHECK(one[1] == sol3); // Inverse filter on scikit-learn==1.5.1 - auto three = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn==1.5.1"), true); + auto three = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn==1.5.1"), + true + ); CHECK(three.size() == 2); CHECK(three[0] == sol0); CHECK(three[1] == sol1); // Filter on scikit-learn<1.5.1 - auto two = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn<1.5.1"), false); + auto two = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn<1.5.1"), + false + ); CHECK(two.size() == 2); CHECK(two[0] == sol0); CHECK(two[1] == sol1); // Filter on build number 0 - auto build = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==0]"), false); + auto build = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn[build_number==0]"), + false + ); CHECK(build.size() == 2); CHECK(build[0] == sol0); CHECK(build[1] == sol2); // Filter on build number 2 - auto build_bis = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==2]"), false); + auto build_bis = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn[build_number==2]"), + false + ); CHECK(build_bis.size() == 1); CHECK(build_bis[0] == sol3); // Filter on build number 3 - auto build_ter = database.filter_candidates(solvables, database.alloc_version_set("scikit-learn[build_number==3]"), false); + auto build_ter = database.filter_candidates( + solvables, + database.alloc_version_set("scikit-learn[build_number==3]"), + false + ); CHECK(build_ter.size() == 0); - } SECTION("Sort solvables increasing order") { @@ -878,7 +996,7 @@ TEST_CASE("solver::resolvo") PackageInfo scikit_learn_ter("scikit-learn", "1.5.1", "py310h981052a_1", 1); auto sol4 = database.alloc_solvable(skl3); - Vector solvables = {sol0, sol1, sol2, sol3, sol4}; + Vector solvables = { sol0, sol1, sol2, sol3, sol4 }; database.sort_candidates(solvables); @@ -887,11 +1005,9 @@ TEST_CASE("solver::resolvo") CHECK(solvables[2] == sol4); CHECK(solvables[3] == sol2); CHECK(solvables[4] == sol0); - } SECTION("Trivial problem") { - PackageDatabase database; // NOTE: the problem can only be solved when two `Solvable` are added to the `PackageDatabase` PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); @@ -923,7 +1039,6 @@ TEST_CASE("solver::resolvo") ); std::cout << "Number of solvables: " << database.solvable_pool.size() << std::endl; - } SECTION("Parse noarch/repodata.json") @@ -939,7 +1054,6 @@ TEST_CASE("solver::resolvo") ); std::cout << "Number of solvables: " << database.solvable_pool.size() << std::endl; - } } From d05f0d209d6b857b264871b83d76629d7c0e69bd Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 8 Jul 2024 18:46:25 +0200 Subject: [PATCH 19/59] test: mlflow=2.12.2 explicit Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 00bb8ee45b..bbc6800224 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -1230,6 +1230,28 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } } + TEST_CASE("mlflow=2.12.2 explicit") { + // See: https://github.com/mamba-org/rattler/issues/684 + std::vector specs_to_install = { "mlflow=2.12.2" }; + + std::vector libsolv_resolution = libsolv_resolve(libsolv_db, specs_to_install); + std::vector resolvo_resolution = resolvo_resolve(resolvo_db, specs_to_install); + + // Check libsolv's PackageInfo against libsolv's + CHECK_EQ(resolvo_resolution.size(), libsolv_resolution.size()); + for (size_t i = 0; i < libsolv_resolution.size(); i++) + { + const PackageInfo& resolvo_package_info = resolvo_resolution[i]; + const PackageInfo& libsolv_package_info = libsolv_resolution[i]; + // Currently something in the parsing of the repodata.json must be different. + // TODO: find the difference and use `PackageInfo::operator==` instead + CHECK_EQ(resolvo_package_info.name, libsolv_package_info.name); + CHECK_EQ(resolvo_package_info.version, libsolv_package_info.version); + CHECK_EQ(resolvo_package_info.build_string, libsolv_package_info.build_string); + } + + } + SECTION("Using YAML environment specification") { From c7d32dcfb55115945914e8d2412ce129b4605bca Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 8 Jul 2024 18:34:57 +0200 Subject: [PATCH 20/59] Debug Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 139 +++++++----------- 1 file changed, 56 insertions(+), 83 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index bbc6800224..51306b3c3e 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -136,7 +136,7 @@ struct PackageDatabase : public DependencyProvider { ::Mapping solvable_pool; // PackageName to Vector - std::unordered_map> name_to_solvable; + std::unordered_map> name_to_solvable; /** * Allocates a new requirement and return the id of the requirement. @@ -350,16 +350,15 @@ struct PackageDatabase : public DependencyProvider { * with the given name is requested. */ Candidates get_candidates(NameId package) override { - // std::cout << "Getting candidates for " << name_pool[package] << std::endl; - Candidates candidates; + std::cout << "DO Getting candidates for " << name_pool[package] << std::endl; + Candidates candidates{}; candidates.favored = nullptr; candidates.locked = nullptr; + std::cout << "DO Assigning candidates for " << name_pool[package] << std::endl; + candidates.candidates = name_to_solvable[package]; + std::cout << "DONE Assigning candidates for " << name_pool[package] << std::endl; - // TODO: implement copy/move-constructor - for (auto& solvable_id : name_to_solvable[package]) { - candidates.candidates.push_back(solvable_id); - } - + std::cout << "DONE Getting candidates for " << name_pool[package] << std::endl; return candidates; } @@ -370,7 +369,6 @@ struct PackageDatabase : public DependencyProvider { * tried. This continues until a solution is found. */ void sort_candidates(Slice solvables) override { - // std::cout << "Sorting candidates" << std::endl; std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { const PackageInfo& package_info_a = solvable_pool[a]; const PackageInfo& package_info_b = solvable_pool[b]; @@ -397,13 +395,13 @@ struct PackageDatabase : public DependencyProvider { VersionSetId version_set_id, bool inverse ) override { + MatchSpec match_spec = version_set_pool[version_set_id]; Vector filtered; if(inverse) { for (auto& solvable_id : candidates) { const PackageInfo& package_info = solvable_pool[solvable_id]; - const MatchSpec match_spec = version_set_pool[version_set_id]; // Is it an appropriate check? Or must another one be crafted? if (!match_spec.contains_except_channel(package_info)) @@ -415,7 +413,6 @@ struct PackageDatabase : public DependencyProvider { for (auto& solvable_id : candidates) { const PackageInfo& package_info = solvable_pool[solvable_id]; - const MatchSpec match_spec = version_set_pool[version_set_id]; // Is it an appropriate check? Or must another one be crafted? if (match_spec.contains_except_channel(package_info)) @@ -424,6 +421,12 @@ struct PackageDatabase : public DependencyProvider { } } } + std::cout << "Keeping " << filtered.size() << " candidates for " << match_spec.str() << ":" << std::endl; + for (auto& solvable_id : filtered) + { + const PackageInfo& package_info = solvable_pool[solvable_id]; + std::cout << " - " << package_info.long_str() << std::endl; + } return filtered; } @@ -432,10 +435,12 @@ struct PackageDatabase : public DependencyProvider { * Returns the dependencies for the specified solvable. */ Dependencies get_dependencies(SolvableId solvable_id) override { + std::cout << "Getting dependencies for " << solvable_id.id << std::endl; const PackageInfo& package_info = solvable_pool[solvable_id]; Dependencies dependencies; + // TODO: do this in O(1) for (auto& dep : package_info.dependencies) { const MatchSpec match_spec = MatchSpec::parse(dep).value(); dependencies.requirements.push_back(version_set_pool[match_spec]); @@ -703,19 +708,19 @@ auto libsolv_db = mamba::solver::libsolv::Database({ /* .channel_alias= */ specs::CondaURL::parse("https://conda.anaconda.org/").value(), }); -const auto repo_linux = libsolv_db.add_repo_from_repodata_json( - "/tmp/linux-64/repodata.json", - "https://conda.anaconda.org/conda-forge/linux-64", - "conda-forge", - libsolv::PipAsPythonDependency::No -); - -const auto repo_noarch = libsolv_db.add_repo_from_repodata_json( - "/tmp/noarch/repodata.json", - "https://conda.anaconda.org/conda-forge/noarch", - "conda-forge", - libsolv::PipAsPythonDependency::Yes -); +//const auto repo_linux = libsolv_db.add_repo_from_repodata_json( +// "/tmp/linux-64/repodata.json", +// "https://conda.anaconda.org/conda-forge/linux-64", +// "conda-forge", +// libsolv::PipAsPythonDependency::No +//); +// +//const auto repo_noarch = libsolv_db.add_repo_from_repodata_json( +// "/tmp/noarch/repodata.json", +// "https://conda.anaconda.org/conda-forge/noarch", +// "conda-forge", +// libsolv::PipAsPythonDependency::Yes +//); PackageDatabase create_resolvo_db() { PackageDatabase resolvo_db; @@ -1230,7 +1235,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } } - TEST_CASE("mlflow=2.12.2 explicit") { + SECTION("mlflow=2.12.2 explicit") { // See: https://github.com/mamba-org/rattler/issues/684 std::vector specs_to_install = { "mlflow=2.12.2" }; @@ -1238,23 +1243,28 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { std::vector resolvo_resolution = resolvo_resolve(resolvo_db, specs_to_install); // Check libsolv's PackageInfo against libsolv's - CHECK_EQ(resolvo_resolution.size(), libsolv_resolution.size()); + REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); for (size_t i = 0; i < libsolv_resolution.size(); i++) { const PackageInfo& resolvo_package_info = resolvo_resolution[i]; const PackageInfo& libsolv_package_info = libsolv_resolution[i]; // Currently something in the parsing of the repodata.json must be different. // TODO: find the difference and use `PackageInfo::operator==` instead - CHECK_EQ(resolvo_package_info.name, libsolv_package_info.name); - CHECK_EQ(resolvo_package_info.version, libsolv_package_info.version); - CHECK_EQ(resolvo_package_info.build_string, libsolv_package_info.build_string); + REQUIRE(resolvo_package_info.name == libsolv_package_info.name); + REQUIRE(resolvo_package_info.version == libsolv_package_info.version); + REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); } - } - SECTION("Using YAML environment specification") { + // "/tmp/unconstrained_tiny_spec3.yaml", + // "/tmp/unconstrained_tiny_spec2.yaml", + // "/tmp/unconstrained_small_spec2.yaml", + // "/tmp/small_spec2.yaml", + // "/tmp/small_spec.yaml" + // "/tmp/spec.yaml" + const fs::u8path& yaml_file = "/tmp/small_spec.yaml"; // "/tmp/spec.yaml" std::cout << "Resolving " << yaml_file << std::endl; mamba::detail::yaml_file_contents res = mamba::detail::read_yaml_file(yaml_file, "linux-64"); @@ -1271,64 +1281,27 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { resolvo::Vector constraints = {}; resolvo::Vector result; - std::cout << "Start with resolvo" << std::endl; - auto tick_resolvo = std::chrono::steady_clock::now(); - String reason = resolvo::solve(resolvo_db, requirements, constraints, result); - auto tack_resolvo = std::chrono::steady_clock::now(); - std::cout << "End with resolvo" << std::endl; - std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; - REQUIRE(reason == ""); + // Print all the dependencies + std::cout << "dependencies" << std::endl; + for (const auto& dep : specs_to_install) + { + std::cout << " - " << dep << std::endl; + } - std::vector resolvo_resolution; - std::transform( - result.begin(), - result.end(), - std::back_inserter(resolvo_resolution), - [&](const resolvo::SolvableId& solvable_id) - { return resolvo_db.solvable_pool[solvable_id]; } - ); + // Add python to the specs to install + // TODO: environments aren't resolved similarly when "python" simply is used. + // specs_to_install.push_back("python>=3.12"); - std::sort( - resolvo_resolution.begin(), - resolvo_resolution.end(), - [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + std::vector libsolv_resolution = libsolv_resolve( + libsolv_db, + specs_to_install ); - - // libsolv's specification and resolution - - Request::job_list jobs; - - std::transform( - specs_to_install.begin(), - specs_to_install.end(), - std::back_inserter(jobs), - [](const std::string& spec) - { return Request::Install{ MatchSpec::parse(spec).value() }; } + std::vector resolvo_resolution = resolvo_resolve( + resolvo_db, + specs_to_install ); - const auto request = Request{ - /* .flags= */ {}, - /* .jobs= */ jobs, - }; - - std::cout << "Start with libsolv" << std::endl; - auto tick_libsolv = std::chrono::steady_clock::now(); - const auto outcome = libsolv::Solver().solve(libsolv_db, request); - auto tack_libsolv = std::chrono::steady_clock::now(); - std::cout << "End with libsolv" << std::endl; - std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() << "ms" << std::endl; - - REQUIRE(outcome.has_value()); - REQUIRE(std::holds_alternative(outcome.value())); - const auto& solution = std::get(outcome.value()); - - std::vector libsolv_resolution = extract_package_to_install(solution); - std::sort( - libsolv_resolution.begin(), - libsolv_resolution.end(), - [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } - ); // Print all the packages from libsolv std::cout << "libsolv resolution" << std::endl; From 6260b22114cfcea3496ee4019371478c03d15aba Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 9 Jul 2024 13:58:56 +0200 Subject: [PATCH 21/59] More tests Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 429 ++++++++++++------ 1 file changed, 287 insertions(+), 142 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 51306b3c3e..78dc47c8e6 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -350,15 +350,10 @@ struct PackageDatabase : public DependencyProvider { * with the given name is requested. */ Candidates get_candidates(NameId package) override { - std::cout << "DO Getting candidates for " << name_pool[package] << std::endl; Candidates candidates{}; candidates.favored = nullptr; candidates.locked = nullptr; - std::cout << "DO Assigning candidates for " << name_pool[package] << std::endl; candidates.candidates = name_to_solvable[package]; - std::cout << "DONE Assigning candidates for " << name_pool[package] << std::endl; - - std::cout << "DONE Getting candidates for " << name_pool[package] << std::endl; return candidates; } @@ -372,16 +367,24 @@ struct PackageDatabase : public DependencyProvider { std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { const PackageInfo& package_info_a = solvable_pool[a]; const PackageInfo& package_info_b = solvable_pool[b]; - // TODO: Add some caching on the version parsing + + // If track features are present, prefer the solvable having the least of them. + if (package_info_a.track_features.size() != package_info_b.track_features.size()) { + return package_info_a.track_features.size() < package_info_b.track_features.size() ; + } + const auto a_version = Version::parse(package_info_a.version).value(); const auto b_version = Version::parse(package_info_b.version).value(); if (a_version != b_version) { return a_version > b_version; } - // TODO: add sorting on track features and other things - return package_info_a.build_number > package_info_b.build_number; + if (package_info_a.build_number != package_info_b.build_number) { + return package_info_a.build_number > package_info_b.build_number; + } + + return package_info_a.timestamp > package_info_b.timestamp; }); } @@ -421,11 +424,11 @@ struct PackageDatabase : public DependencyProvider { } } } - std::cout << "Keeping " << filtered.size() << " candidates for " << match_spec.str() << ":" << std::endl; + // std::cout << "Keeping " << filtered.size() << " candidates for " << match_spec.str() << ":" << std::endl; for (auto& solvable_id : filtered) { const PackageInfo& package_info = solvable_pool[solvable_id]; - std::cout << " - " << package_info.long_str() << std::endl; + // std::cout << " - " << package_info.long_str() << std::endl; } return filtered; @@ -435,7 +438,7 @@ struct PackageDatabase : public DependencyProvider { * Returns the dependencies for the specified solvable. */ Dependencies get_dependencies(SolvableId solvable_id) override { - std::cout << "Getting dependencies for " << solvable_id.id << std::endl; + // std::cout << "Getting dependencies for " << solvable_id.id << std::endl; const PackageInfo& package_info = solvable_pool[solvable_id]; Dependencies dependencies; @@ -703,24 +706,40 @@ auto extract_package_to_install(const Solution& solution) -> std::vector resolvo_resolve( - PackageDatabase& database, - const std::vector& specs -) { - // resolvo's specification and resolution - resolvo::Vector requirements; - for (const auto& spec : specs) - { - requirements.push_back(resolvo_db.alloc_version_set(spec)); - } - - resolvo::Vector constraints = {}; - resolvo::Vector result; - - std::cout << "Start with resolvo" << std::endl; - auto tick_resolvo = std::chrono::steady_clock::now(); - String reason = resolvo::solve(resolvo_db, requirements, constraints, result); - auto tack_resolvo = std::chrono::steady_clock::now(); - std::cout << "End with resolvo" << std::endl; - std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; - - std::cout << "Resolvo's Reason: " << reason << std::endl; - - std::vector resolvo_resolution; - std::transform( - result.begin(), - result.end(), - std::back_inserter(resolvo_resolution), - [&](const resolvo::SolvableId& solvable_id) - { return resolvo_db.solvable_pool[solvable_id]; } - ); - - std::sort( - resolvo_resolution.begin(), - resolvo_resolution.end(), - [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } - ); - return resolvo_resolution; -} - std::vector libsolv_resolve( mamba::solver::libsolv::Database& db, const std::vector& specs @@ -821,16 +800,63 @@ std::vector libsolv_resolve( std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() << "ms" << std::endl; REQUIRE(outcome.has_value()); - REQUIRE(std::holds_alternative(outcome.value())); - const auto& solution = std::get(outcome.value()); - - std::vector libsolv_resolution = extract_package_to_install(solution); - std::sort( - libsolv_resolution.begin(), - libsolv_resolution.end(), - [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } - ); - return libsolv_resolution; + if (std::holds_alternative(outcome.value())) + { + const auto& solution = std::get(outcome.value()); + + std::vector libsolv_resolution = extract_package_to_install(solution); + std::sort( + libsolv_resolution.begin(), + libsolv_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + return libsolv_resolution; + } + return {}; +} + + +std::vector resolvo_resolve( + PackageDatabase& database, + const std::vector& specs +) { + // resolvo's specification and resolution + resolvo::Vector requirements; + for (const auto& spec : specs) + { + requirements.push_back(resolvo_db.alloc_version_set(spec)); + } + + resolvo::Vector constraints = {}; + resolvo::Vector result; + + std::cout << "Start with resolvo" << std::endl; + auto tick_resolvo = std::chrono::steady_clock::now(); + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); + auto tack_resolvo = std::chrono::steady_clock::now(); + std::cout << "End with resolvo" << std::endl; + std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; + + if (reason == "") + { + std::vector resolvo_resolution; + for(auto solvable_id : result) + { + PackageInfo package_info = resolvo_db.solvable_pool[solvable_id]; + // Skip virtual package (i.e. whose `package_info.name` starts with "__") + if (package_info.name.find("__") != 0) { + resolvo_resolution.push_back(package_info); + } + } + + std::sort( + resolvo_resolution.begin(), + resolvo_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); + return resolvo_resolution; + } + return {}; } @@ -998,18 +1024,63 @@ TEST_CASE("solver::resolvo") PackageInfo skl3("scikit-learn", "1.5.0", "py310h981052a_2", 2); auto sol3 = database.alloc_solvable(skl3); - PackageInfo scikit_learn_ter("scikit-learn", "1.5.1", "py310h981052a_1", 1); - auto sol4 = database.alloc_solvable(skl3); + PackageInfo skl4("scikit-learn", "1.5.1", "py310h981052a_1", 1); + auto sol4 = database.alloc_solvable(skl4); Vector solvables = { sol0, sol1, sol2, sol3, sol4 }; database.sort_candidates(solvables); - CHECK(solvables[0] == sol1); - CHECK(solvables[1] == sol3); - CHECK(solvables[2] == sol4); - CHECK(solvables[3] == sol2); - CHECK(solvables[4] == sol0); + REQUIRE(solvables[0] == sol0); + REQUIRE(solvables[1] == sol2); + REQUIRE(solvables[2] == sol4); + REQUIRE(solvables[3] == sol3); + REQUIRE(solvables[4] == sol1); + } + + SECTION("Sort solvables (build number only)") + { + PackageDatabase database; + + PackageInfo skl0("scikit-learn", "1.5.0", "py310h981052a_0", 0); + auto sol0 = database.alloc_solvable(skl0); + + PackageInfo skl1("scikit-learn", "1.5.0", "py310h981052a_3", 3); + auto sol1 = database.alloc_solvable(skl1); + + PackageInfo skl2("scikit-learn", "1.5.0", "py310h981052a_2", 2); + auto sol2 = database.alloc_solvable(skl2); + + PackageInfo skl3("scikit-learn", "1.5.0", "py310h981052a_1", 1); + auto sol3 = database.alloc_solvable(skl3); + + PackageInfo skl4("scikit-learn", "1.5.0", "py310h981052a_4", 4); + auto sol4 = database.alloc_solvable(skl4); + + PackageInfo skl5("scikit-learn", "1.5.0", "py310h981052a_5", 5); + skl5.timestamp = 1337; + auto sol5 = database.alloc_solvable(skl5); + + PackageInfo skl6("scikit-learn", "1.5.0", "py310h981052a_5", 5); + skl6.timestamp = 42; + auto sol6 = database.alloc_solvable(skl6); + + PackageInfo skl7("scikit-learn", "1.5.0", "py310h981052a_5", 5); + skl7.timestamp = 2000; + auto sol7 = database.alloc_solvable(skl7); + + Vector solvables = { sol0, sol1, sol2, sol3, sol4, sol5, sol6, sol7 }; + + database.sort_candidates(solvables); + + REQUIRE(solvables[0] == sol7); + REQUIRE(solvables[1] == sol5); + REQUIRE(solvables[2] == sol6); + REQUIRE(solvables[3] == sol4); + REQUIRE(solvables[4] == sol1); + REQUIRE(solvables[5] == sol2); + REQUIRE(solvables[6] == sol3); + REQUIRE(solvables[7] == sol0); } SECTION("Trivial problem") { @@ -1119,7 +1190,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { PackageInfo("_libgcc_mutex", "0.1", "conda_forge", 0), PackageInfo("python_abi", "3.10", "4_cp310", 0), PackageInfo("ld_impl_linux-64", "2.40", "hf3520f5_7", 0), - PackageInfo("ca-certificates", "2024.6.2", "hbcca054_0", 0), + PackageInfo("ca-certificates", "2024.7.4", "hbcca054_0", 0), PackageInfo("libgomp", "14.1.0", "h77fa898_0", 0), PackageInfo("_openmp_mutex", "4.5", "2_gnu", 0), PackageInfo("libgcc-ng", "14.1.0", "h77fa898_0", 0), @@ -1138,7 +1209,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { PackageInfo("libsqlite", "3.46.0", "hde9e2c9_0", 0), PackageInfo("readline", "8.2", "h8228510_1", 0), PackageInfo("libgfortran-ng", "14.1.0", "h69a702a_0", 0), - PackageInfo("libopenblas", "0.3.27", "pthreads_h413a1c8_0", 0), + PackageInfo("libopenblas", "0.3.27", "pthreads_hac2b453_1", 0), PackageInfo("libblas", "3.9.0", "22_linux64_openblas", 0), PackageInfo("libcblas", "3.9.0", "22_linux64_openblas", 0), PackageInfo("liblapack", "3.9.0", "22_linux64_openblas", 0), @@ -1150,7 +1221,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { PackageInfo("threadpoolctl", "3.5.0", "pyhc1e730c_0", 0), PackageInfo("joblib", "1.4.2", "pyhd8ed1ab_0", 0), PackageInfo("numpy", "1.26.4", "py310hb13e2d6_0", 0), - PackageInfo("scipy", "1.14.0", "py310h93e2701_0", 0), + PackageInfo("scipy", "1.14.0", "py310h93e2701_1", 0), PackageInfo("scikit-learn", "1.5.0", "py310h981052a_1", 1) }; @@ -1235,88 +1306,110 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } } - SECTION("mlflow=2.12.2 explicit") { - // See: https://github.com/mamba-org/rattler/issues/684 - std::vector specs_to_install = { "mlflow=2.12.2" }; - - std::vector libsolv_resolution = libsolv_resolve(libsolv_db, specs_to_install); - std::vector resolvo_resolution = resolvo_resolve(resolvo_db, specs_to_install); - - // Check libsolv's PackageInfo against libsolv's - REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); - for (size_t i = 0; i < libsolv_resolution.size(); i++) + SECTION("mamba-org/rattler/issues/684") + { + for (const std::vector& specs_to_install : std::initializer_list> { + {"mlflow=2.12.2"}, + {"orange3=3.36.2"}, + {"ray-dashboard=2.6.3"}, + {"ray-default=2.6.3"}, + {"spark-nlp=5.1.2"}, + {"spyder=5.5.1"}, + {"streamlit-faker=0.0.2"} + }) { - const PackageInfo& resolvo_package_info = resolvo_resolution[i]; - const PackageInfo& libsolv_package_info = libsolv_resolution[i]; - // Currently something in the parsing of the repodata.json must be different. - // TODO: find the difference and use `PackageInfo::operator==` instead - REQUIRE(resolvo_package_info.name == libsolv_package_info.name); - REQUIRE(resolvo_package_info.version == libsolv_package_info.version); - REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); - } - } + // See: https://github.com/mamba-org/rattler/issues/684 + std::vector libsolv_resolution = libsolv_resolve( + libsolv_db, + specs_to_install + ); + + // Print all the packages from libsolv + std::cout << "libsolv resolution:" << std::endl; + for (const auto& package_info : libsolv_resolution) + { + std::cout << " - " << package_info.long_str() << std::endl; + } - SECTION("Using YAML environment specification") - { - // "/tmp/unconstrained_tiny_spec3.yaml", - // "/tmp/unconstrained_tiny_spec2.yaml", - // "/tmp/unconstrained_small_spec2.yaml", - // "/tmp/small_spec2.yaml", - // "/tmp/small_spec.yaml" - // "/tmp/spec.yaml" + std::cout << std::endl; + std::vector resolvo_resolution = resolvo_resolve( + resolvo_db, + specs_to_install + ); - const fs::u8path& yaml_file = "/tmp/small_spec.yaml"; // "/tmp/spec.yaml" - std::cout << "Resolving " << yaml_file << std::endl; - mamba::detail::yaml_file_contents res = mamba::detail::read_yaml_file(yaml_file, "linux-64"); + // Print all the packages from resolvo + std::cout << "resolvo resolution:" << std::endl; + for (const auto& package_info : resolvo_resolution) + { + std::cout << " - " << package_info.long_str() << std::endl; + } - std::vector specs_to_install = res.dependencies; + REQUIRE(resolvo_resolution.size() > 0); + REQUIRE(libsolv_resolution.size() > 0); - // resolvo's specification and resolution - resolvo::Vector requirements; - for (const auto& spec : specs_to_install) - { - requirements.push_back(resolvo_db.alloc_version_set(spec)); + // Check libsolv's PackageInfo against libsolv's + REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); + for (size_t i = 0; i < std::min(resolvo_resolution.size(), libsolv_resolution.size()); i++) + { + const PackageInfo& resolvo_package_info = resolvo_resolution[i]; + const PackageInfo& libsolv_package_info = libsolv_resolution[i]; + // Currently something in the parsing of the repodata.json must be different. + // TODO: find the difference and use `PackageInfo::operator==` instead + REQUIRE(resolvo_package_info.name == libsolv_package_info.name); +// REQUIRE(resolvo_package_info.version == libsolv_package_info.version); +// REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); + } } + } - resolvo::Vector constraints = {}; - resolvo::Vector result; + SECTION("Consistency with libsolv: Celery & Dash") + { + std::vector specs_to_install = { + // TODO: when python >=3.12 is unpinned, environment aren't identical + "python", + "celery", + "dash", + "dash-core-components", + "dash-html-components", + "dash-table" + }; // Print all the dependencies - std::cout << "dependencies" << std::endl; + std::cout << "Specification to install:" << std::endl; for (const auto& dep : specs_to_install) { std::cout << " - " << dep << std::endl; } - // Add python to the specs to install - // TODO: environments aren't resolved similarly when "python" simply is used. - // specs_to_install.push_back("python>=3.12"); - std::vector libsolv_resolution = libsolv_resolve( libsolv_db, specs_to_install ); - std::vector resolvo_resolution = resolvo_resolve( - resolvo_db, - specs_to_install - ); - // Print all the packages from libsolv - std::cout << "libsolv resolution" << std::endl; + std::cout << "libsolv resolution:" << std::endl; for (const auto& package_info : libsolv_resolution) { - std::cout << package_info.long_str() << std::endl; + std::cout << " - " << package_info.long_str() << std::endl; } + std::cout << std::endl; + + std::vector resolvo_resolution = resolvo_resolve( + resolvo_db, + specs_to_install + ); + // Print all the packages from resolvo - std::cout << "resolvo resolution" << std::endl; + std::cout << "resolvo resolution:" << std::endl; for (const auto& package_info : resolvo_resolution) { - std::cout << package_info.long_str() << std::endl; + std::cout << " - " << package_info.long_str() << std::endl; } + std::cout << std::endl; + // Check libsolv's PackageInfo against libsolv's REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); for (size_t i = 0; i < libsolv_resolution.size(); i++) @@ -1330,4 +1423,56 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); } } + + SECTION("Consistency with libsolv: robin-env specifications") { + for (const std::string& specification: { + // See: https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 + "rubin-env", + "rubin-env-rsp", + "rubin-env-developer" + }) + { + std::cout << "Resolving " << specification << std::endl; + + std::vector specs_to_install = {specification}; + + std::vector libsolv_resolution = libsolv_resolve( + libsolv_db, + specs_to_install + ); + std::vector resolvo_resolution = resolvo_resolve( + resolvo_db, + specs_to_install + ); + + // Print all the packages from libsolv + std::cout << "libsolv resolution:" << std::endl; + for (const auto& package_info : libsolv_resolution) + { + std::cout << " - " << package_info.long_str() << std::endl; + } + + std::cout << std::endl; + + // Print all the packages from resolvo + std::cout << "resolvo resolution:" << std::endl; + for (const auto& package_info : resolvo_resolution) + { + std::cout << " - " << package_info.long_str() << std::endl; + } + + // Check libsolv's PackageInfo against libsolv's + REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); + for (size_t i = 0; i < libsolv_resolution.size(); i++) + { + const PackageInfo& resolvo_package_info = resolvo_resolution[i]; + const PackageInfo& libsolv_package_info = libsolv_resolution[i]; + // Currently something in the parsing of the repodata.json must be different. + // TODO: find the difference and use `PackageInfo::operator==` instead + REQUIRE(resolvo_package_info.name == libsolv_package_info.name); + REQUIRE(resolvo_package_info.version == libsolv_package_info.version); + REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); + } + } + } } From c78107e1440838dfec72c987011e7bb69589b5c6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 10 Jul 2024 12:19:37 +0200 Subject: [PATCH 22/59] Complete ordering of solvables Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 95 ++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 78dc47c8e6..5804fbaa7e 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include @@ -138,6 +139,10 @@ struct PackageDatabase : public DependencyProvider { // PackageName to Vector std::unordered_map> name_to_solvable; + // VersionSetId to max version + // TODO use `SolvableId` instead of `std::pair`? + std::unordered_map> version_set_to_max_version_and_track_features_numbers; + /** * Allocates a new requirement and return the id of the requirement. */ @@ -357,6 +362,41 @@ struct PackageDatabase : public DependencyProvider { return candidates; } + std::pair find_highest_version( + VersionSetId version_set_id + ) { + // If the version set has already been computed, return it. + if(version_set_to_max_version_and_track_features_numbers.find(version_set_id) != version_set_to_max_version_and_track_features_numbers.end()) { + return version_set_to_max_version_and_track_features_numbers[version_set_id]; + } + + const MatchSpec match_spec = version_set_pool[version_set_id]; + + const std::string& name = match_spec.name().str(); + + auto name_id = name_pool.alloc(String{name}); + + auto solvables = name_to_solvable[name_id]; + + auto filtered = filter_candidates(solvables, version_set_id, false); + + Version max_version = Version(); + size_t max_n_track_features = 0; + + for(auto& solvable_id : filtered) { + const PackageInfo& package_info = solvable_pool[solvable_id]; + const auto version = Version::parse(package_info.version).value(); + if(version > max_version) { + max_version = version; + max_n_track_features = package_info.track_features.size(); + } + } + + auto val = std::make_pair(max_version, max_n_track_features); + version_set_to_max_version_and_track_features_numbers[version_set_id] = val; + return val; + } + /** * Sort the specified solvables based on which solvable to try first. The * solver will iteratively try to select the highest version. If a @@ -384,6 +424,48 @@ struct PackageDatabase : public DependencyProvider { return package_info_a.build_number > package_info_b.build_number; } + // Compare the dependencies of the variants. + std::unordered_map a_deps; + std::unordered_map b_deps; + for(auto dep_a: package_info_a.dependencies) { + // TODO: have a VersionID to NameID mapping instead + MatchSpec ms = MatchSpec::parse(dep_a).value(); + const std::string& name = ms.name().str(); + auto name_id = name_pool.alloc(String{name}); + + a_deps[name_id] = version_set_pool[ms]; + } + for(auto dep_b: package_info_b.dependencies) { + // TODO: have a VersionID to NameID mapping instead + MatchSpec ms = MatchSpec::parse(dep_b).value(); + const std::string& name = ms.name().str(); + auto name_id = name_pool.alloc(String{name}); + + b_deps[name_id] = version_set_pool[ms]; + } + + auto ordering_score = 0; + for (auto [name_id, version_set_id] : a_deps) { + if (b_deps.find(name_id) != b_deps.end()) { + auto [a_version, a_n_track_features] = find_highest_version(version_set_id); + auto [b_version, b_n_track_features] = find_highest_version(b_deps[name_id]); + + // Favor the solvable with higher versions of their dependencies + if (a_version != b_version) { + ordering_score += a_version > b_version ? 1 : -1; + } + + // Highly penalize the solvable if a dependencies has more track features + if (a_n_track_features != b_n_track_features) { + ordering_score += a_n_track_features > b_n_track_features ? -100 : 100; + } + } + } + + if(ordering_score != 0) { + return ordering_score > 0; + } + return package_info_a.timestamp > package_info_b.timestamp; }); } @@ -1362,12 +1444,21 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } } + SECTION("Find the highest version of hypothesis") + { + // Some builds of hypothesis depends on attrs and vice-versa + // We test that this complete correctly. + auto vid = resolvo_db.alloc_version_set("hypothesis"); + auto [version, n_track_features] = resolvo_db.find_highest_version(vid); + REQUIRE(n_track_features == 0); + std::cout << "Version: " << version.str() << std::endl; + REQUIRE(version > Version::parse("6.105.1").value()); + } + SECTION("Consistency with libsolv: Celery & Dash") { std::vector specs_to_install = { - // TODO: when python >=3.12 is unpinned, environment aren't identical - "python", "celery", "dash", "dash-core-components", From 3864c845b8a70119edb86cefe71da65809e2c682 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 12 Jul 2024 11:37:47 +0200 Subject: [PATCH 23/59] Case for ordering on track features Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 5804fbaa7e..9065e9ab36 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -381,18 +381,21 @@ struct PackageDatabase : public DependencyProvider { auto filtered = filter_candidates(solvables, version_set_id, false); Version max_version = Version(); - size_t max_n_track_features = 0; + size_t max_version_n_track_features = 0; for(auto& solvable_id : filtered) { const PackageInfo& package_info = solvable_pool[solvable_id]; const auto version = Version::parse(package_info.version).value(); + if(version == max_version) { + max_version_n_track_features = std::min(max_version_n_track_features, package_info.track_features.size()); + } if(version > max_version) { max_version = version; - max_n_track_features = package_info.track_features.size(); + max_version_n_track_features = package_info.track_features.size(); } } - auto val = std::make_pair(max_version, max_n_track_features); + auto val = std::make_pair(max_version, max_version_n_track_features); version_set_to_max_version_and_track_features_numbers[version_set_id] = val; return val; } @@ -520,18 +523,30 @@ struct PackageDatabase : public DependencyProvider { * Returns the dependencies for the specified solvable. */ Dependencies get_dependencies(SolvableId solvable_id) override { - // std::cout << "Getting dependencies for " << solvable_id.id << std::endl; const PackageInfo& package_info = solvable_pool[solvable_id]; + // std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; Dependencies dependencies; // TODO: do this in O(1) for (auto& dep : package_info.dependencies) { + // std::cout << "Parsing dep " << dep << std::endl; const MatchSpec match_spec = MatchSpec::parse(dep).value(); dependencies.requirements.push_back(version_set_pool[match_spec]); } for (auto& constr : package_info.constrains) { - const MatchSpec match_spec = MatchSpec::parse(constr).value(); + // std::cout << "Parsing constr " << constr << std::endl; + // if constr contain " == " replace it with "==" + std::string constr2 = constr; + while (constr2.find(" == ") != std::string::npos) + { + constr2 = constr2.replace(constr2.find(" == "), 4, "=="); + } + while (constr2.find(" >= ") != std::string::npos) + { + constr2 = constr2.replace(constr2.find(" >= "), 4, ">="); + } + const MatchSpec match_spec = MatchSpec::parse(constr2).value(); dependencies.constrains.push_back(version_set_pool[match_spec]); } @@ -1391,13 +1406,15 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { SECTION("mamba-org/rattler/issues/684") { for (const std::vector& specs_to_install : std::initializer_list> { - {"mlflow=2.12.2"}, - {"orange3=3.36.2"}, - {"ray-dashboard=2.6.3"}, - {"ray-default=2.6.3"}, - {"spark-nlp=5.1.2"}, - {"spyder=5.5.1"}, - {"streamlit-faker=0.0.2"} + // TODO: Currently does not probably due to the ordering of the packages on track features + {"arrow-cpp", "abseil-cpp"}, +// {"mlflow=2.12.2"}, +// {"orange3=3.36.2"}, +// {"ray-dashboard=2.6.3"}, +// {"ray-default=2.6.3"}, +// {"spark-nlp=5.1.2"}, +// {"spyder=5.5.1"}, +// {"streamlit-faker=0.0.2"} }) { // See: https://github.com/mamba-org/rattler/issues/684 @@ -1438,8 +1455,8 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { // Currently something in the parsing of the repodata.json must be different. // TODO: find the difference and use `PackageInfo::operator==` instead REQUIRE(resolvo_package_info.name == libsolv_package_info.name); -// REQUIRE(resolvo_package_info.version == libsolv_package_info.version); -// REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); + REQUIRE(resolvo_package_info.version == libsolv_package_info.version); + REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); } } } @@ -1518,9 +1535,10 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { SECTION("Consistency with libsolv: robin-env specifications") { for (const std::string& specification: { // See: https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 - "rubin-env", - "rubin-env-rsp", - "rubin-env-developer" + "rubin-env-nosysroot", +// "rubin-env", +// "rubin-env-rsp", +// "rubin-env-developer" }) { std::cout << "Resolving " << specification << std::endl; From b1f764f467a8c36460e459bb5a5710b4e9edffcc Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 25 Jul 2024 14:23:02 +0200 Subject: [PATCH 24/59] Remove old tests Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 9065e9ab36..f9b22df158 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -263,6 +263,17 @@ struct PackageDatabase : public DependencyProvider { alloc_version_set(constr); } +// const size_t n_track_features = package_info.track_features.size(); +// if(n_track_features > 0) +// { +// std::cout << "PackageInfo has " << package_info.long_str() << " has " << n_track_features << " track features" << std::endl; +// for (auto tf :package_info.track_features) +// { +// // Add the track feature to the Name and String pools +// std::cout << " - " << tf < filtered; +// std::cout << "Candidates to filter " << match_spec.str() << std::endl; +// +// for(auto& solvable_id : candidates) { +// const PackageInfo& package_info = solvable_pool[solvable_id]; +// std::cout << " - " << package_info.long_str() << std::endl; +// } + if(inverse) { for (auto& solvable_id : candidates) { @@ -509,12 +527,12 @@ struct PackageDatabase : public DependencyProvider { } } } - // std::cout << "Keeping " << filtered.size() << " candidates for " << match_spec.str() << ":" << std::endl; - for (auto& solvable_id : filtered) - { - const PackageInfo& package_info = solvable_pool[solvable_id]; - // std::cout << " - " << package_info.long_str() << std::endl; - } +// std::cout << "Filtered candidates for " << match_spec.str() << std::endl; +// +// for(auto& solvable_id : filtered) { +// const PackageInfo& package_info = solvable_pool[solvable_id]; +// std::cout << " - " << package_info.long_str() << std::endl; +// } return filtered; } @@ -1406,15 +1424,15 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { SECTION("mamba-org/rattler/issues/684") { for (const std::vector& specs_to_install : std::initializer_list> { - // TODO: Currently does not probably due to the ordering of the packages on track features - {"arrow-cpp", "abseil-cpp"}, -// {"mlflow=2.12.2"}, -// {"orange3=3.36.2"}, -// {"ray-dashboard=2.6.3"}, -// {"ray-default=2.6.3"}, -// {"spark-nlp=5.1.2"}, -// {"spyder=5.5.1"}, -// {"streamlit-faker=0.0.2"} + // TODO: Currently does not probably due to the ordering of the packages on track features + {"arrow-cpp", "libabseil"}, + {"mlflow=2.12.2"}, + {"orange3=3.36.2"}, + {"ray-dashboard=2.6.3"}, + {"ray-default=2.6.3"}, + {"spark-nlp=5.1.2"}, + {"spyder=5.5.1"}, + {"streamlit-faker=0.0.2"} }) { // See: https://github.com/mamba-org/rattler/issues/684 @@ -1581,6 +1599,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE(resolvo_package_info.name == libsolv_package_info.name); REQUIRE(resolvo_package_info.version == libsolv_package_info.version); REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); + } } } From 34f91a0aea3c94f44305d8fae9c4fb23246afc52 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 25 Jul 2024 14:39:46 +0200 Subject: [PATCH 25/59] Combine test cases Also clear a bit. Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 66 +++---------------- 1 file changed, 8 insertions(+), 58 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index f9b22df158..a7597c315c 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -1421,10 +1421,10 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } } - SECTION("mamba-org/rattler/issues/684") + SECTION("Known hard specifications") { for (const std::vector& specs_to_install : std::initializer_list> { - // TODO: Currently does not probably due to the ordering of the packages on track features + // See: https://github.com/mamba-org/rattler/issues/684 {"arrow-cpp", "libabseil"}, {"mlflow=2.12.2"}, {"orange3=3.36.2"}, @@ -1432,7 +1432,12 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { {"ray-default=2.6.3"}, {"spark-nlp=5.1.2"}, {"spyder=5.5.1"}, - {"streamlit-faker=0.0.2"} + {"streamlit-faker=0.0.2"}, + // See: https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 + {"rubin-env-nosysroot"}, + {"rubin-env"}, + {"rubin-env-rsp"}, + {"rubin-env-developer"} }) { // See: https://github.com/mamba-org/rattler/issues/684 @@ -1490,7 +1495,6 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE(version > Version::parse("6.105.1").value()); } - SECTION("Consistency with libsolv: Celery & Dash") { std::vector specs_to_install = { @@ -1549,58 +1553,4 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); } } - - SECTION("Consistency with libsolv: robin-env specifications") { - for (const std::string& specification: { - // See: https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 - "rubin-env-nosysroot", -// "rubin-env", -// "rubin-env-rsp", -// "rubin-env-developer" - }) - { - std::cout << "Resolving " << specification << std::endl; - - std::vector specs_to_install = {specification}; - - std::vector libsolv_resolution = libsolv_resolve( - libsolv_db, - specs_to_install - ); - std::vector resolvo_resolution = resolvo_resolve( - resolvo_db, - specs_to_install - ); - - // Print all the packages from libsolv - std::cout << "libsolv resolution:" << std::endl; - for (const auto& package_info : libsolv_resolution) - { - std::cout << " - " << package_info.long_str() << std::endl; - } - - std::cout << std::endl; - - // Print all the packages from resolvo - std::cout << "resolvo resolution:" << std::endl; - for (const auto& package_info : resolvo_resolution) - { - std::cout << " - " << package_info.long_str() << std::endl; - } - - // Check libsolv's PackageInfo against libsolv's - REQUIRE(resolvo_resolution.size() == libsolv_resolution.size()); - for (size_t i = 0; i < libsolv_resolution.size(); i++) - { - const PackageInfo& resolvo_package_info = resolvo_resolution[i]; - const PackageInfo& libsolv_package_info = libsolv_resolution[i]; - // Currently something in the parsing of the repodata.json must be different. - // TODO: find the difference and use `PackageInfo::operator==` instead - REQUIRE(resolvo_package_info.name == libsolv_package_info.name); - REQUIRE(resolvo_package_info.version == libsolv_package_info.version); - REQUIRE(resolvo_package_info.build_string == libsolv_package_info.build_string); - - } - } - } } From ade5030ed01612c41a30dc6a0c7a9f74e381069b Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 17 Feb 2025 10:12:19 +0100 Subject: [PATCH 26/59] Require, simplify, reformat, new sklearn example Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 853 +++++++++++------- 1 file changed, 517 insertions(+), 336 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index a7597c315c..6e8802d7ee 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -11,12 +11,10 @@ #include #include #include - #include -#include "mamba/api/install.hpp" // for parsing YAML specs - -#include "mamba/core/util.hpp" // for LockFile +#include "mamba/api/install.hpp" // for parsing YAML specs +#include "mamba/core/util.hpp" // for LockFile #include "mamba/specs/channel.hpp" #include "mamba/specs/package_info.hpp" @@ -36,37 +34,45 @@ using namespace mamba::solver; using namespace resolvo; template <> -struct std::hash { - std::size_t operator()(const VersionSetId& id) const { +struct std::hash +{ + std::size_t operator()(const VersionSetId& id) const + { return std::hash{}(id.id); } }; template <> -struct std::hash { - std::size_t operator()(const SolvableId& id) const { +struct std::hash +{ + std::size_t operator()(const SolvableId& id) const + { return std::hash{}(id.id); } }; template <> -struct std::hash { - std::size_t operator()(const NameId& id) const { +struct std::hash +{ + std::size_t operator()(const NameId& id) const + { return std::hash{}(id.id); } }; template <> -struct std::hash { - std::size_t operator()(const StringId& id) const { +struct std::hash +{ + std::size_t operator()(const StringId& id) const + { return std::hash{}(id.id); } }; - // Create a template Pool class that maps a key to a set of values template -struct Mapping { +struct Mapping +{ Mapping() = default; ~Mapping() = default; @@ -74,11 +80,13 @@ struct Mapping { * Adds the value to the Mapping and returns its associated id. If the * value is already in the Mapping, returns the id associated with it. */ - ID alloc(T value) { - if (auto element = value_to_id.find(value); element != value_to_id.end()) { + ID alloc(T value) + { + if (auto element = value_to_id.find(value); element != value_to_id.end()) + { return element->second; } - auto id = ID{static_cast(id_to_value.size())}; + auto id = ID{ static_cast(id_to_value.size()) }; id_to_value[id] = value; value_to_id[value] = id; return id; @@ -87,44 +95,119 @@ struct Mapping { /** * Returns the value associated with the given id. */ - T operator[](ID id) { return id_to_value[id]; } + T operator[](ID id) + { + return id_to_value[id]; + } /** * Returns the id associated with the given value. */ - ID operator[](T value) { return value_to_id[value]; } + ID operator[](T value) + { + return value_to_id[value]; + } // Iterator for the Mapping - auto begin() { return id_to_value.begin(); } - auto end() { return id_to_value.end(); } - auto begin() const { return id_to_value.begin(); } - auto end() const { return id_to_value.end(); } - auto cbegin() { return id_to_value.cbegin(); } - auto cend() { return id_to_value.cend(); } - auto cbegin() const { return id_to_value.cbegin(); } - auto cend() const { return id_to_value.cend(); } - auto find(T value) { return value_to_id.find(value); } - - auto begin_ids() { return value_to_id.begin(); } - auto end_ids() { return value_to_id.end(); } - auto begin_ids() const { return value_to_id.begin(); } - auto end_ids() const { return value_to_id.end(); } - auto cbegin_ids() { return value_to_id.cbegin(); } - auto cend_ids() { return value_to_id.cend(); } - auto cbegin_ids() const { return value_to_id.cbegin(); } - auto cend_ids() const { return value_to_id.cend(); } - - auto size() const { return id_to_value.size(); } + auto begin() + { + return id_to_value.begin(); + } + + auto end() + { + return id_to_value.end(); + } + + auto begin() const + { + return id_to_value.begin(); + } + + auto end() const + { + return id_to_value.end(); + } + + auto cbegin() + { + return id_to_value.cbegin(); + } + + auto cend() + { + return id_to_value.cend(); + } + + auto cbegin() const + { + return id_to_value.cbegin(); + } + + auto cend() const + { + return id_to_value.cend(); + } + + auto find(T value) + { + return value_to_id.find(value); + } + + auto begin_ids() + { + return value_to_id.begin(); + } + + auto end_ids() + { + return value_to_id.end(); + } + + auto begin_ids() const + { + return value_to_id.begin(); + } + + auto end_ids() const + { + return value_to_id.end(); + } + + auto cbegin_ids() + { + return value_to_id.cbegin(); + } + + auto cend_ids() + { + return value_to_id.cend(); + } + + auto cbegin_ids() const + { + return value_to_id.cbegin(); + } + + auto cend_ids() const + { + return value_to_id.cend(); + } + + auto size() const + { + return id_to_value.size(); + } private: + std::unordered_map value_to_id; std::unordered_map id_to_value; }; - -struct PackageDatabase : public DependencyProvider { - +struct PackageDatabase : public DependencyProvider +{ virtual ~PackageDatabase() = default; ::Mapping name_pool; @@ -141,25 +224,27 @@ struct PackageDatabase : public DependencyProvider { // VersionSetId to max version // TODO use `SolvableId` instead of `std::pair`? - std::unordered_map> version_set_to_max_version_and_track_features_numbers; + std::unordered_map> + version_set_to_max_version_and_track_features_numbers; /** * Allocates a new requirement and return the id of the requirement. */ - VersionSetId alloc_version_set( - std::string_view raw_match_spec - ) { + VersionSetId alloc_version_set(std::string_view raw_match_spec) + { std::string raw_match_spec_str = std::string(raw_match_spec); // Replace all " v" with simply " " to work around the `v` prefix in some version strings - // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in `infom2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` + // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in + // `inform2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` while (raw_match_spec_str.find(" v") != std::string::npos) { raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(" v"), 2, " "); } // Remove any presence of selector on python version in the match spec - // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in `infowillow-1.6.3-pyhd8ed1ab_0.conda` - for(const auto specifier: {"=py", "py", ">=py", "<=py", "!=py"}) + // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in + // `infowillow-1.6.3-pyhd8ed1ab_0.conda` + for (const auto specifier : { "=py", "py", ">=py", "<=py", "!=py" }) { while (raw_match_spec_str.find(specifier) != std::string::npos) { @@ -167,7 +252,8 @@ struct PackageDatabase : public DependencyProvider { } } // Remove any white space between version - // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` + // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in + // `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` while (raw_match_spec_str.find(", ") != std::string::npos) { raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(", "), 2, ","); @@ -176,12 +262,12 @@ struct PackageDatabase : public DependencyProvider { // TODO: skip allocation for now if "*.*" is in the match spec if (raw_match_spec_str.find("*.*") != std::string::npos) { - return VersionSetId{0}; + return VersionSetId{ 0 }; } - // NOTE: works around `openblas 0.2.18|0.2.18.*.` from `dlib==19.0=np110py27_blas_openblas_200` - // If contains "|", split on it and recurse + // NOTE: works around `openblas 0.2.18|0.2.18.*.` from + // `dlib==19.0=np110py27_blas_openblas_200` If contains "|", split on it and recurse if (raw_match_spec_str.find("|") != std::string::npos) { std::vector match_specs; @@ -205,21 +291,29 @@ struct PackageDatabase : public DependencyProvider { alloc_version_set(ms); } // Placeholder return value - return VersionSetId{0}; + return VersionSetId{ 0 }; } // NOTE: This works around some improperly encoded `constrains` in the test data, e.g.: - // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit >= 10.2" - // `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" - // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: ">=4.5.2" + // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit + // >= 10.2" `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded + // `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" + // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: + // ">=4.5.2" // Remove any with space after the binary operators - for(const std::string& op : {">=", "<=", "==", ">", "<", "!=", "=", "=="}) { + for (const std::string& op : { ">=", "<=", "==", ">", "<", "!=", "=", "==" }) + { const std::string& bad_op = op + " "; - while (raw_match_spec_str.find(bad_op) != std::string::npos) { - raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op + raw_match_spec_str.substr(raw_match_spec_str.find(bad_op) + bad_op.size()); + while (raw_match_spec_str.find(bad_op) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op + + raw_match_spec_str.substr( + raw_match_spec_str.find(bad_op) + bad_op.size() + ); } // If start with binary operator, prepend NONE - if (raw_match_spec_str.find(op) == 0) { + if (raw_match_spec_str.find(op) == 0) + { raw_match_spec_str = "NONE " + raw_match_spec_str; } } @@ -230,52 +324,54 @@ struct PackageDatabase : public DependencyProvider { // Add name to the Name and String pools const std::string name = match_spec.name().str(); - name_pool.alloc(String{name}); - string_pool.alloc(String{name}); + name_pool.alloc(String{ name }); + string_pool.alloc(String{ name }); // Add the MatchSpec's string representation to the Name and String pools const std::string match_spec_str = match_spec.str(); - name_pool.alloc(String{match_spec_str}); - string_pool.alloc(String{match_spec_str}); + name_pool.alloc(String{ match_spec_str }); + string_pool.alloc(String{ match_spec_str }); return id; } - SolvableId alloc_solvable( - PackageInfo package_info - ) { + SolvableId alloc_solvable(PackageInfo package_info) + { // Add the solvable to the solvable pool auto id = solvable_pool.alloc(package_info); // Add name to the Name and String pools const std::string name = package_info.name; - name_pool.alloc(String{name}); - string_pool.alloc(String{name}); + name_pool.alloc(String{ name }); + string_pool.alloc(String{ name }); // Add the long string representation of the package to the Name and String pools const std::string long_str = package_info.long_str(); - name_pool.alloc(String{long_str}); - string_pool.alloc(String{long_str}); + name_pool.alloc(String{ long_str }); + string_pool.alloc(String{ long_str }); - for (auto& dep : package_info.dependencies) { + for (auto& dep : package_info.dependencies) + { alloc_version_set(dep); } - for (auto& constr : package_info.constrains) { + for (auto& constr : package_info.constrains) + { alloc_version_set(constr); } -// const size_t n_track_features = package_info.track_features.size(); -// if(n_track_features > 0) -// { -// std::cout << "PackageInfo has " << package_info.long_str() << " has " << n_track_features << " track features" << std::endl; -// for (auto tf :package_info.track_features) -// { -// // Add the track feature to the Name and String pools -// std::cout << " - " << tf < 0) + // { + // std::cout << "PackageInfo has " << package_info.long_str() << " has " << + // n_track_features << " track features" << std::endl; for (auto tf + // :package_info.track_features) + // { + // // Add the track feature to the Name and String pools + // std::cout << " - " << tf < solvable) override { + String display_merged_solvables(Slice solvable) override + { std::string result; - for (auto& solvable_id : solvable) { + for (auto& solvable_id : solvable) + { result += solvable_pool[solvable_id].long_str(); } - return String{result}; + return String{ result }; } /** * Returns an object that can be used to display the given name in a * user-friendly way. */ - String display_name(NameId name) override { + String display_name(NameId name) override + { return name_pool[name]; } @@ -330,15 +431,17 @@ struct PackageDatabase : public DependencyProvider { * The name of the package should *not* be included in the display. Where * appropriate, this information is added. */ - String display_version_set(VersionSetId version_set) override { + String display_version_set(VersionSetId version_set) override + { const MatchSpec match_spec = version_set_pool[version_set]; - return String{match_spec.str()}; + return String{ match_spec.str() }; } /** * Returns the string representation of the specified string. */ - String display_string(StringId string) override { + String display_string(StringId string) override + { return string_pool[string]; } @@ -346,26 +449,30 @@ struct PackageDatabase : public DependencyProvider { * Returns the name of the package that the specified version set is * associated with. */ - NameId version_set_name(VersionSetId version_set_id) override { + NameId version_set_name(VersionSetId version_set_id) override + { const MatchSpec match_spec = version_set_pool[version_set_id]; - // std::cout << "Getting name id for version_set_id " << match_spec.name().str() << std::endl; - return name_pool[String{match_spec.name().str()}]; + // std::cout << "Getting name id for version_set_id " << match_spec.name().str() << + // std::endl; + return name_pool[String{ match_spec.name().str() }]; } /** * Returns the name of the package for the given solvable. */ - NameId solvable_name(SolvableId solvable_id) override { + NameId solvable_name(SolvableId solvable_id) override + { const PackageInfo& package_info = solvable_pool[solvable_id]; - //std::cout << "Getting name id for solvable " << package_info.long_str() << std::endl; - return name_pool[String{package_info.name}]; + // std::cout << "Getting name id for solvable " << package_info.long_str() << std::endl; + return name_pool[String{ package_info.name }]; } /** * Obtains a list of solvables that should be considered when a package * with the given name is requested. */ - Candidates get_candidates(NameId package) override { + Candidates get_candidates(NameId package) override + { Candidates candidates{}; candidates.favored = nullptr; candidates.locked = nullptr; @@ -373,11 +480,12 @@ struct PackageDatabase : public DependencyProvider { return candidates; } - std::pair find_highest_version( - VersionSetId version_set_id - ) { + std::pair find_highest_version(VersionSetId version_set_id) + { // If the version set has already been computed, return it. - if(version_set_to_max_version_and_track_features_numbers.find(version_set_id) != version_set_to_max_version_and_track_features_numbers.end()) { + if (version_set_to_max_version_and_track_features_numbers.find(version_set_id) + != version_set_to_max_version_and_track_features_numbers.end()) + { return version_set_to_max_version_and_track_features_numbers[version_set_id]; } @@ -385,7 +493,7 @@ struct PackageDatabase : public DependencyProvider { const std::string& name = match_spec.name().str(); - auto name_id = name_pool.alloc(String{name}); + auto name_id = name_pool.alloc(String{ name }); auto solvables = name_to_solvable[name_id]; @@ -394,13 +502,19 @@ struct PackageDatabase : public DependencyProvider { Version max_version = Version(); size_t max_version_n_track_features = 0; - for(auto& solvable_id : filtered) { + for (auto& solvable_id : filtered) + { const PackageInfo& package_info = solvable_pool[solvable_id]; const auto version = Version::parse(package_info.version).value(); - if(version == max_version) { - max_version_n_track_features = std::min(max_version_n_track_features, package_info.track_features.size()); + if (version == max_version) + { + max_version_n_track_features = std::min( + max_version_n_track_features, + package_info.track_features.size() + ); } - if(version > max_version) { + if (version > max_version) + { max_version = version; max_version_n_track_features = package_info.track_features.size(); } @@ -417,71 +531,89 @@ struct PackageDatabase : public DependencyProvider { * conflict is found with the highest version the next version is * tried. This continues until a solution is found. */ - void sort_candidates(Slice solvables) override { - std::sort(solvables.begin(), solvables.end(), [&](const SolvableId& a, const SolvableId& b) { - const PackageInfo& package_info_a = solvable_pool[a]; - const PackageInfo& package_info_b = solvable_pool[b]; - - // If track features are present, prefer the solvable having the least of them. - if (package_info_a.track_features.size() != package_info_b.track_features.size()) { - return package_info_a.track_features.size() < package_info_b.track_features.size() ; - } + void sort_candidates(Slice solvables) override + { + std::sort( + solvables.begin(), + solvables.end(), + [&](const SolvableId& a, const SolvableId& b) + { + const PackageInfo& package_info_a = solvable_pool[a]; + const PackageInfo& package_info_b = solvable_pool[b]; - const auto a_version = Version::parse(package_info_a.version).value(); - const auto b_version = Version::parse(package_info_b.version).value(); + // If track features are present, prefer the solvable having the least of them. + if (package_info_a.track_features.size() != package_info_b.track_features.size()) + { + return package_info_a.track_features.size() + < package_info_b.track_features.size(); + } - if (a_version != b_version) { - return a_version > b_version; - } + const auto a_version = Version::parse(package_info_a.version).value(); + const auto b_version = Version::parse(package_info_b.version).value(); - if (package_info_a.build_number != package_info_b.build_number) { - return package_info_a.build_number > package_info_b.build_number; - } + if (a_version != b_version) + { + return a_version > b_version; + } - // Compare the dependencies of the variants. - std::unordered_map a_deps; - std::unordered_map b_deps; - for(auto dep_a: package_info_a.dependencies) { - // TODO: have a VersionID to NameID mapping instead - MatchSpec ms = MatchSpec::parse(dep_a).value(); - const std::string& name = ms.name().str(); - auto name_id = name_pool.alloc(String{name}); + if (package_info_a.build_number != package_info_b.build_number) + { + return package_info_a.build_number > package_info_b.build_number; + } - a_deps[name_id] = version_set_pool[ms]; - } - for(auto dep_b: package_info_b.dependencies) { - // TODO: have a VersionID to NameID mapping instead - MatchSpec ms = MatchSpec::parse(dep_b).value(); - const std::string& name = ms.name().str(); - auto name_id = name_pool.alloc(String{name}); + // Compare the dependencies of the variants. + std::unordered_map a_deps; + std::unordered_map b_deps; + for (auto dep_a : package_info_a.dependencies) + { + // TODO: have a VersionID to NameID mapping instead + MatchSpec ms = MatchSpec::parse(dep_a).value(); + const std::string& name = ms.name().str(); + auto name_id = name_pool.alloc(String{ name }); - b_deps[name_id] = version_set_pool[ms]; - } + a_deps[name_id] = version_set_pool[ms]; + } + for (auto dep_b : package_info_b.dependencies) + { + // TODO: have a VersionID to NameID mapping instead + MatchSpec ms = MatchSpec::parse(dep_b).value(); + const std::string& name = ms.name().str(); + auto name_id = name_pool.alloc(String{ name }); - auto ordering_score = 0; - for (auto [name_id, version_set_id] : a_deps) { - if (b_deps.find(name_id) != b_deps.end()) { - auto [a_version, a_n_track_features] = find_highest_version(version_set_id); - auto [b_version, b_n_track_features] = find_highest_version(b_deps[name_id]); + b_deps[name_id] = version_set_pool[ms]; + } - // Favor the solvable with higher versions of their dependencies - if (a_version != b_version) { - ordering_score += a_version > b_version ? 1 : -1; + auto ordering_score = 0; + for (auto [name_id, version_set_id] : a_deps) + { + if (b_deps.find(name_id) != b_deps.end()) + { + auto [a_tf_version, a_n_track_features] = find_highest_version(version_set_id); + auto [b_tf_version, b_n_track_features] = find_highest_version(b_deps[name_id] + ); + + // Favor the solvable with higher versions of their dependencies + if (a_tf_version != b_tf_version) + { + ordering_score += a_tf_version > b_tf_version ? 1 : -1; + } + + // Highly penalize the solvable if a dependencies has more track features + if (a_n_track_features != b_n_track_features) + { + ordering_score += a_n_track_features > b_n_track_features ? -100 : 100; + } } + } - // Highly penalize the solvable if a dependencies has more track features - if (a_n_track_features != b_n_track_features) { - ordering_score += a_n_track_features > b_n_track_features ? -100 : 100; - } + if (ordering_score != 0) + { + return ordering_score > 0; } - } - if(ordering_score != 0) { - return ordering_score > 0; + return package_info_a.timestamp > package_info_b.timestamp; } - - return package_info_a.timestamp > package_info_b.timestamp; - }); + ); } /** @@ -489,22 +621,21 @@ struct PackageDatabase : public DependencyProvider { * version set or if `inverse` is true, the solvables that do *not* match * the version set. */ - Vector filter_candidates( - Slice candidates, - VersionSetId version_set_id, - bool inverse - ) override { + Vector + filter_candidates(Slice candidates, VersionSetId version_set_id, bool inverse) override + { MatchSpec match_spec = version_set_pool[version_set_id]; Vector filtered; -// std::cout << "Candidates to filter " << match_spec.str() << std::endl; -// -// for(auto& solvable_id : candidates) { -// const PackageInfo& package_info = solvable_pool[solvable_id]; -// std::cout << " - " << package_info.long_str() << std::endl; -// } + // std::cout << "Candidates to filter " << match_spec.str() << std::endl; + // + // for(auto& solvable_id : candidates) { + // const PackageInfo& package_info = solvable_pool[solvable_id]; + // std::cout << " - " << package_info.long_str() << std::endl; + // } - if(inverse) { + if (inverse) + { for (auto& solvable_id : candidates) { const PackageInfo& package_info = solvable_pool[solvable_id]; @@ -515,7 +646,9 @@ struct PackageDatabase : public DependencyProvider { filtered.push_back(solvable_id); } } - } else { + } + else + { for (auto& solvable_id : candidates) { const PackageInfo& package_info = solvable_pool[solvable_id]; @@ -527,12 +660,12 @@ struct PackageDatabase : public DependencyProvider { } } } -// std::cout << "Filtered candidates for " << match_spec.str() << std::endl; -// -// for(auto& solvable_id : filtered) { -// const PackageInfo& package_info = solvable_pool[solvable_id]; -// std::cout << " - " << package_info.long_str() << std::endl; -// } + // std::cout << "Filtered candidates for " << match_spec.str() << std::endl; + // + // for(auto& solvable_id : filtered) { + // const PackageInfo& package_info = solvable_pool[solvable_id]; + // std::cout << " - " << package_info.long_str() << std::endl; + // } return filtered; } @@ -540,19 +673,22 @@ struct PackageDatabase : public DependencyProvider { /** * Returns the dependencies for the specified solvable. */ - Dependencies get_dependencies(SolvableId solvable_id) override { + Dependencies get_dependencies(SolvableId solvable_id) override + { const PackageInfo& package_info = solvable_pool[solvable_id]; // std::cout << "Getting dependencies for " << package_info.long_str() << std::endl; Dependencies dependencies; // TODO: do this in O(1) - for (auto& dep : package_info.dependencies) { + for (auto& dep : package_info.dependencies) + { // std::cout << "Parsing dep " << dep << std::endl; const MatchSpec match_spec = MatchSpec::parse(dep).value(); dependencies.requirements.push_back(version_set_pool[match_spec]); } - for (auto& constr : package_info.constrains) { + for (auto& constr : package_info.constrains) + { // std::cout << "Parsing constr " << constr << std::endl; // if constr contain " == " replace it with "==" std::string constr2 = constr; @@ -570,11 +706,11 @@ struct PackageDatabase : public DependencyProvider { return dependencies; } - }; // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` -auto lsplit_track_features(std::string_view features) +auto +lsplit_track_features(std::string_view features) { constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; auto [_, tail] = util::lstrip_if_parts(features, is_sep); @@ -582,13 +718,15 @@ auto lsplit_track_features(std::string_view features) } // TODO: factorise with the implementation from `set_solvable` in `mamba/solver/libsolv/helpers.cpp` -bool parse_packageinfo_json( +bool +parse_packageinfo_json( const std::string_view& filename, const simdjson::dom::element& pkg, const CondaURL& repo_url, const std::string& channel_id, PackageDatabase& database - ) { +) +{ PackageInfo package_info; package_info.channel = channel_id; @@ -747,7 +885,8 @@ bool parse_packageinfo_json( return true; } -void parse_repodata_json( +void +parse_repodata_json( PackageDatabase& database, const fs::u8path& filename, const std::string& repo_url, @@ -769,8 +908,7 @@ void parse_repodata_json( // Get `base_url` in case 'repodata_version': 2 // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md auto base_url = repo_url; - if (auto repodata_version = repodata["repodata_version"].get_int64(); - !repodata_version.error()) + if (auto repodata_version = repodata["repodata_version"].get_int64(); !repodata_version.error()) { if (repodata_version.value_unsafe() == 2) { @@ -786,8 +924,7 @@ void parse_repodata_json( .value(); auto signatures = std::optional(std::nullopt); - if (auto maybe_sigs = repodata["signatures"].get_object(); - !maybe_sigs.error() && verify_artifacts) + if (auto maybe_sigs = repodata["signatures"].get_object(); !maybe_sigs.error() && verify_artifacts) { signatures = std::move(maybe_sigs).value(); } @@ -814,15 +951,20 @@ void parse_repodata_json( } // from `src/test_solver.cpp` -auto find_actions_with_name(const Solution& solution, std::string_view name) -> std::vector; -auto find_actions(const Solution& solution) -> std::vector; -auto extract_package_to_install(const Solution& solution) -> std::vector; - +auto +find_actions_with_name(const Solution& solution, std::string_view name) + -> std::vector; +auto +find_actions(const Solution& solution) -> std::vector; +auto +extract_package_to_install(const Solution& solution) -> std::vector; // wget https://conda.anaconda.org/conda-forge/linux-64/repodata.json // wget https://conda.anaconda.org/conda-forge/noarch/repodata.json -mamba::solver::libsolv::Database create_libsolv_db() { +mamba::solver::libsolv::Database +create_libsolv_db() +{ auto libsolv_db = mamba::solver::libsolv::Database({ /* .platforms= */ { "linux-64", "noarch" }, /* .channel_alias= */ specs::CondaURL::parse("https://conda.anaconda.org/").value(), @@ -856,7 +998,9 @@ mamba::solver::libsolv::Database create_libsolv_db() { return libsolv_db; }; -PackageDatabase create_resolvo_db() { +PackageDatabase +create_resolvo_db() +{ PackageDatabase resolvo_db; parse_repodata_json( @@ -886,10 +1030,9 @@ PackageDatabase create_resolvo_db() { mamba::solver::libsolv::Database libsolv_db = create_libsolv_db(); PackageDatabase resolvo_db = create_resolvo_db(); -std::vector libsolv_resolve( - mamba::solver::libsolv::Database& db, - const std::vector& specs -) { +std::vector +libsolv_resolve(const std::vector& specs) +{ // libsolv's specification and resolution Request::job_list jobs; @@ -898,8 +1041,7 @@ std::vector libsolv_resolve( specs.begin(), specs.end(), std::back_inserter(jobs), - [](const std::string& spec) - { return Request::Install{ MatchSpec::parse(spec).value() }; } + [](const std::string& spec) { return Request::Install{ MatchSpec::parse(spec).value() }; } ); const auto request = Request{ @@ -912,7 +1054,10 @@ std::vector libsolv_resolve( const auto outcome = libsolv::Solver().solve(libsolv_db, request); auto tack_libsolv = std::chrono::steady_clock::now(); std::cout << "End with libsolv" << std::endl; - std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() << "ms" << std::endl; + std::cout + << "Elapsed time: " + << std::chrono::duration_cast(tack_libsolv - tick_libsolv).count() + << "ms" << std::endl; REQUIRE(outcome.has_value()); if (std::holds_alternative(outcome.value())) @@ -930,11 +1075,9 @@ std::vector libsolv_resolve( return {}; } - -std::vector resolvo_resolve( - PackageDatabase& database, - const std::vector& specs -) { +std::vector +resolvo_resolve(const std::vector& specs) +{ // resolvo's specification and resolution resolvo::Vector requirements; for (const auto& spec : specs) @@ -950,16 +1093,20 @@ std::vector resolvo_resolve( String reason = resolvo::solve(resolvo_db, requirements, constraints, result); auto tack_resolvo = std::chrono::steady_clock::now(); std::cout << "End with resolvo" << std::endl; - std::cout << "Elapsed time: " << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() << "ms" << std::endl; + std::cout + << "Elapsed time: " + << std::chrono::duration_cast(tack_resolvo - tick_resolvo).count() + << "ms" << std::endl; if (reason == "") { std::vector resolvo_resolution; - for(auto solvable_id : result) + for (auto solvable_id : result) { PackageInfo package_info = resolvo_db.solvable_pool[solvable_id]; // Skip virtual package (i.e. whose `package_info.name` starts with "__") - if (package_info.name.find("__") != 0) { + if (package_info.name.find("__") != 0) + { resolvo_resolution.push_back(package_info); } } @@ -974,14 +1121,14 @@ std::vector resolvo_resolve( return {}; } - TEST_CASE("solver::resolvo") { using namespace specs::match_spec_literals; using PackageInfo = PackageInfo; - SECTION("Addition of PackageInfo to PackageDatabase") { + SECTION("Addition of PackageInfo to PackageDatabase") + { PackageDatabase database; PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); @@ -992,44 +1139,45 @@ TEST_CASE("solver::resolvo") auto solvable = database.alloc_solvable(scikit_learn); - CHECK(solvable.id == 0); - CHECK(database.solvable_pool[solvable].name == "scikit-learn"); - CHECK(database.solvable_pool[solvable].version == "1.5.0"); - CHECK(database.solvable_pool[solvable].build_string == "py310h981052a_0"); - CHECK(database.solvable_pool[solvable].build_number == 0); + REQUIRE(solvable.id == 0); + REQUIRE(database.solvable_pool[solvable].name == "scikit-learn"); + REQUIRE(database.solvable_pool[solvable].version == "1.5.0"); + REQUIRE(database.solvable_pool[solvable].build_string == "py310h981052a_0"); + REQUIRE(database.solvable_pool[solvable].build_number == 0); auto deps = database.get_dependencies(solvable); - CHECK(deps.requirements.size() == 4); - CHECK(deps.constrains.size() == 0); + REQUIRE(deps.requirements.size() == 4); + REQUIRE(deps.constrains.size() == 0); - CHECK( + REQUIRE( database.version_set_pool[deps.requirements[0]].str() == "numpy[version=\">=1.20.0,<2.0a0\"]" ); - CHECK( + REQUIRE( database.version_set_pool[deps.requirements[1]].str() == "scipy[version=\">=1.6.0,<2.0a0\"]" ); - CHECK( + REQUIRE( database.version_set_pool[deps.requirements[2]].str() == "joblib[version=\">=1.0.1,<2.0a0\"]" ); - CHECK( + REQUIRE( database.version_set_pool[deps.requirements[3]].str() == "threadpoolctl[version=\">=2.1.0,<3.0a0\"]" ); - CHECK(database.name_pool.find(String{ "scikit-learn" }) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{ "numpy" }) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{ "scipy" }) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{ "joblib" }) != database.name_pool.end_ids()); - CHECK(database.name_pool.find(String{ "threadpoolctl" }) != database.name_pool.end_ids()); - - CHECK(database.string_pool.find(String{ "scikit-learn" }) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{ "numpy" }) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{ "scipy" }) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{ "joblib" }) != database.string_pool.end_ids()); - CHECK(database.string_pool.find(String{ "threadpoolctl" }) != database.string_pool.end_ids()); + REQUIRE(database.name_pool.find(String{ "scikit-learn" }) != database.name_pool.end_ids()); + REQUIRE(database.name_pool.find(String{ "numpy" }) != database.name_pool.end_ids()); + REQUIRE(database.name_pool.find(String{ "scipy" }) != database.name_pool.end_ids()); + REQUIRE(database.name_pool.find(String{ "joblib" }) != database.name_pool.end_ids()); + REQUIRE(database.name_pool.find(String{ "threadpoolctl" }) != database.name_pool.end_ids()); + + REQUIRE(database.string_pool.find(String{ "scikit-learn" }) != database.string_pool.end_ids()); + REQUIRE(database.string_pool.find(String{ "numpy" }) != database.string_pool.end_ids()); + REQUIRE(database.string_pool.find(String{ "scipy" }) != database.string_pool.end_ids()); + REQUIRE(database.string_pool.find(String{ "joblib" }) != database.string_pool.end_ids()); + REQUIRE(database.string_pool.find(String{ "threadpoolctl" }) != database.string_pool.end_ids()); } - SECTION("Filter solvables") { + SECTION("Filter solvables") + { PackageDatabase database; PackageInfo skl0("scikit-learn", "1.4.0", "py310h981052a_0", 0); @@ -1052,11 +1200,11 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn"), false ); - CHECK(all.size() == 4); - CHECK(all[0] == sol0); - CHECK(all[1] == sol1); - CHECK(all[2] == sol2); - CHECK(all[3] == sol3); + REQUIRE(all.size() == 4); + REQUIRE(all[0] == sol0); + REQUIRE(all[1] == sol1); + REQUIRE(all[2] == sol2); + REQUIRE(all[3] == sol3); // Inverse filter on scikit-learn auto none = database.filter_candidates( @@ -1064,7 +1212,7 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn"), true ); - CHECK(none.size() == 0); + REQUIRE(none.size() == 0); // Filter on scikit-learn==1.5.1 auto one = database.filter_candidates( @@ -1072,9 +1220,9 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn==1.5.1"), false ); - CHECK(one.size() == 2); - CHECK(one[0] == sol2); - CHECK(one[1] == sol3); + REQUIRE(one.size() == 2); + REQUIRE(one[0] == sol2); + REQUIRE(one[1] == sol3); // Inverse filter on scikit-learn==1.5.1 auto three = database.filter_candidates( @@ -1082,9 +1230,9 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn==1.5.1"), true ); - CHECK(three.size() == 2); - CHECK(three[0] == sol0); - CHECK(three[1] == sol1); + REQUIRE(three.size() == 2); + REQUIRE(three[0] == sol0); + REQUIRE(three[1] == sol1); // Filter on scikit-learn<1.5.1 auto two = database.filter_candidates( @@ -1092,9 +1240,9 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn<1.5.1"), false ); - CHECK(two.size() == 2); - CHECK(two[0] == sol0); - CHECK(two[1] == sol1); + REQUIRE(two.size() == 2); + REQUIRE(two[0] == sol0); + REQUIRE(two[1] == sol1); // Filter on build number 0 auto build = database.filter_candidates( @@ -1102,9 +1250,9 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn[build_number==0]"), false ); - CHECK(build.size() == 2); - CHECK(build[0] == sol0); - CHECK(build[1] == sol2); + REQUIRE(build.size() == 2); + REQUIRE(build[0] == sol0); + REQUIRE(build[1] == sol2); // Filter on build number 2 auto build_bis = database.filter_candidates( @@ -1112,8 +1260,8 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn[build_number==2]"), false ); - CHECK(build_bis.size() == 1); - CHECK(build_bis[0] == sol3); + REQUIRE(build_bis.size() == 1); + REQUIRE(build_bis[0] == sol3); // Filter on build number 3 auto build_ter = database.filter_candidates( @@ -1121,10 +1269,11 @@ TEST_CASE("solver::resolvo") database.alloc_version_set("scikit-learn[build_number==3]"), false ); - CHECK(build_ter.size() == 0); + REQUIRE(build_ter.size() == 0); } - SECTION("Sort solvables increasing order") { + SECTION("Sort solvables increasing order") + { PackageDatabase database; PackageInfo skl0("scikit-learn", "1.5.2", "py310h981052a_0", 0); @@ -1198,9 +1347,11 @@ TEST_CASE("solver::resolvo") REQUIRE(solvables[7] == sol0); } - SECTION("Trivial problem") { + SECTION("Trivial problem") + { PackageDatabase database; - // NOTE: the problem can only be solved when two `Solvable` are added to the `PackageDatabase` + // NOTE: the problem can only be solved when two `Solvable` are added to the + // `PackageDatabase` PackageInfo scikit_learn("scikit-learn", "1.5.0", "py310h981052a_0", 0); database.alloc_solvable(scikit_learn); @@ -1212,9 +1363,9 @@ TEST_CASE("solver::resolvo") resolvo::Vector result; String reason = resolvo::solve(database, requirements, constraints, result); - CHECK(reason == ""); - CHECK(result.size() == 1); - CHECK(database.solvable_pool[result[0]] == scikit_learn); + REQUIRE(reason == ""); + REQUIRE(result.size() == 1); + REQUIRE(database.solvable_pool[result[0]] == scikit_learn); } SECTION("Parse linux-64/repodata.json") @@ -1248,7 +1399,8 @@ TEST_CASE("solver::resolvo") } } -TEST_CASE("Test consistency with libsolv (environment creation)") { +TEST_CASE("Test consistency with libsolv (environment creation)") +{ using namespace specs::match_spec_literals; using PackageInfo = PackageInfo; @@ -1268,13 +1420,13 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE_FALSE(solution.actions.empty()); // Numpy is last because of topological sort - CHECK(std::holds_alternative(solution.actions.back())); + REQUIRE(std::holds_alternative(solution.actions.back())); REQUIRE(std::get(solution.actions.back()).install.name == "numpy"); REQUIRE(find_actions_with_name(solution, "numpy").size() == 1); const auto python_actions = find_actions_with_name(solution, "python"); REQUIRE(python_actions.size() == 1); - CHECK(std::holds_alternative(python_actions.front())); + REQUIRE(std::holds_alternative(python_actions.front())); resolvo::Vector requirements = { resolvo_db.alloc_version_set("numpy"), @@ -1285,19 +1437,53 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { String reason = resolvo::solve(resolvo_db, requirements, constraints, result); REQUIRE(reason == ""); - REQUIRE(result.size() == 29); + REQUIRE(result.size() == 31); REQUIRE(resolvo_db.solvable_pool[result[0]].name == "numpy"); } - SECTION("scikit-learn explicit") { + SECTION("scikit-learn") + { + const auto request = Request{ + /* .flags= */ {}, + /* .jobs= */ { Request::Install{ "scikit-learn"_ms } }, + }; + + const auto outcome = libsolv::Solver().solve(libsolv_db, request); + + REQUIRE(outcome.has_value()); + REQUIRE(std::holds_alternative(outcome.value())); + const auto& solution = std::get(outcome.value()); + + REQUIRE_FALSE(solution.actions.empty()); + + // scikit-learn is last because of topological sort + REQUIRE(std::holds_alternative(solution.actions.back())); + REQUIRE(std::get(solution.actions.back()).install.name == "scikit-learn"); + REQUIRE(find_actions_with_name(solution, "scikit-learn").size() == 1); + + const auto python_actions = find_actions_with_name(solution, "scikit-learn"); + REQUIRE(python_actions.size() == 1); + REQUIRE(std::holds_alternative(python_actions.front())); + + resolvo::Vector requirements = { + resolvo_db.alloc_version_set("scikit-learn"), + }; + + resolvo::Vector constraints = {}; + resolvo::Vector result; + String reason = resolvo::solve(resolvo_db, requirements, constraints, result); + + REQUIRE(reason == ""); + REQUIRE(result.size() == 36); + REQUIRE(resolvo_db.solvable_pool[result[0]].name == "scikit-learn"); + } + SECTION("scikit-learn explicit") + { std::vector specs_to_install = { - "python[version=\">=3.10,<3.11.0a0\"]", - "pip", - "scikit-learn[version=\">=1.0.0,<1.6a0\"]", - "numpy[version=\">=1.20.0,<2.0a0\"]", - "scipy[version=\">=1.10.0,<1.15a0\"]", - "joblib[version=\">=1.0.1,<2.0a0\"]", + "python[version=\">=3.10,<3.11.0a0\"]", "pip", + "scikit-learn[version=\">=1.0.0,<1.6a0\"]", "numpy[version=\">=1.20.0,<2.0a0\"]", + "scipy[version=\">=1.10.0,<1.15a0\"]", "joblib[version=\">=1.0.1,<2.0a0\"]", "threadpoolctl[version=\">=2.1.0,<3.6a0\"]", }; @@ -1340,9 +1526,11 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { PackageInfo("scikit-learn", "1.5.0", "py310h981052a_1", 1) }; - std::sort(known_resolution.begin(), known_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { - return a.name < b.name; - }); + std::sort( + known_resolution.begin(), + known_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); // libsolv's specification and resolution @@ -1352,9 +1540,8 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { specs_to_install.begin(), specs_to_install.end(), std::back_inserter(jobs), - [](const std::string& spec) { - return Request::Install{MatchSpec::parse(spec).value()}; - } + [](const std::string& spec) + { return Request::Install{ MatchSpec::parse(spec).value() }; } ); const auto request = Request{ @@ -1371,13 +1558,16 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { REQUIRE(solution.actions.size() == known_resolution.size()); std::vector libsolv_resolution = extract_package_to_install(solution); - std::sort(libsolv_resolution.begin(), libsolv_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { - return a.name < b.name; - }); + std::sort( + libsolv_resolution.begin(), + libsolv_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); // resolvo's specification and resolution resolvo::Vector requirements; - for (const auto& spec : specs_to_install) { + for (const auto& spec : specs_to_install) + { requirements.push_back(resolvo_db.alloc_version_set(spec)); } @@ -1393,17 +1583,19 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { result.begin(), result.end(), std::back_inserter(resolvo_resolution), - [&](const resolvo::SolvableId& solvable_id) { - return resolvo_db.solvable_pool[solvable_id]; - } + [&](const resolvo::SolvableId& solvable_id) + { return resolvo_db.solvable_pool[solvable_id]; } ); - std::sort(resolvo_resolution.begin(), resolvo_resolution.end(), [&](const PackageInfo& a, const PackageInfo& b) { - return a.name < b.name; - }); + std::sort( + resolvo_resolution.begin(), + resolvo_resolution.end(), + [&](const PackageInfo& a, const PackageInfo& b) { return a.name < b.name; } + ); // Check libsolv's PackageInfo against the know resolution - for (size_t i = 0; i < libsolv_resolution.size(); i++) { + for (size_t i = 0; i < libsolv_resolution.size(); i++) + { const PackageInfo& package_info = libsolv_resolution[i]; const PackageInfo& known_package_info = known_resolution[i]; REQUIRE(package_info.name == known_package_info.name); @@ -1412,7 +1604,8 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } // Check resolvo's PackageInfo against the know resolution - for (size_t i = 0; i < resolvo_resolution.size(); i++) { + for (size_t i = 0; i < resolvo_resolution.size(); i++) + { const PackageInfo& package_info = resolvo_resolution[i]; const PackageInfo& known_package_info = known_resolution[i]; REQUIRE(package_info.name == known_package_info.name); @@ -1423,28 +1616,27 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { SECTION("Known hard specifications") { - for (const std::vector& specs_to_install : std::initializer_list> { - // See: https://github.com/mamba-org/rattler/issues/684 - {"arrow-cpp", "libabseil"}, - {"mlflow=2.12.2"}, - {"orange3=3.36.2"}, - {"ray-dashboard=2.6.3"}, - {"ray-default=2.6.3"}, - {"spark-nlp=5.1.2"}, - {"spyder=5.5.1"}, - {"streamlit-faker=0.0.2"}, - // See: https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 - {"rubin-env-nosysroot"}, - {"rubin-env"}, - {"rubin-env-rsp"}, - {"rubin-env-developer"} - }) + for (const std::vector& specs_to_install : + std::initializer_list>{ + // See: https://github.com/mamba-org/rattler/issues/684 + // {"arrow-cpp", "libabseil"}, + // {"mlflow=2.12.2"}, + // {"orange3=3.36.2"}, + // {"ray-dashboard=2.6.3"}, + // {"ray-default=2.6.3"}, + // {"spark-nlp=5.1.2"}, + // {"spyder=5.5.1"}, + // {"streamlit-faker=0.0.2"}, + // // See: + // https://github.com/conda-forge/rubinenv-feedstock/blob/main/recipe/meta.yaml#L45-L191 + // {"rubin-env-nosysroot"}, + // {"rubin-env"}, + // {"rubin-env-rsp"}, + // {"rubin-env-developer"} + }) { // See: https://github.com/mamba-org/rattler/issues/684 - std::vector libsolv_resolution = libsolv_resolve( - libsolv_db, - specs_to_install - ); + std::vector libsolv_resolution = libsolv_resolve(specs_to_install); // Print all the packages from libsolv std::cout << "libsolv resolution:" << std::endl; @@ -1454,10 +1646,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { } std::cout << std::endl; - std::vector resolvo_resolution = resolvo_resolve( - resolvo_db, - specs_to_install - ); + std::vector resolvo_resolution = resolvo_resolve(specs_to_install); // Print all the packages from resolvo std::cout << "resolvo resolution:" << std::endl; @@ -1497,13 +1686,11 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { SECTION("Consistency with libsolv: Celery & Dash") { - std::vector specs_to_install = { - "celery", - "dash", - "dash-core-components", - "dash-html-components", - "dash-table" - }; + std::vector specs_to_install = { "celery", + "dash", + "dash-core-components", + "dash-html-components", + "dash-table" }; // Print all the dependencies std::cout << "Specification to install:" << std::endl; @@ -1512,10 +1699,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { std::cout << " - " << dep << std::endl; } - std::vector libsolv_resolution = libsolv_resolve( - libsolv_db, - specs_to_install - ); + std::vector libsolv_resolution = libsolv_resolve(specs_to_install); // Print all the packages from libsolv std::cout << "libsolv resolution:" << std::endl; @@ -1526,10 +1710,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") { std::cout << std::endl; - std::vector resolvo_resolution = resolvo_resolve( - resolvo_db, - specs_to_install - ); + std::vector resolvo_resolution = resolvo_resolve(specs_to_install); // Print all the packages from resolvo std::cout << "resolvo resolution:" << std::endl; From 3f84945b53be61313d01cb1f75cb534da568e9bc Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 17 Feb 2025 10:19:50 +0100 Subject: [PATCH 27/59] Update scikit-learn explicit known resolution Signed-off-by: Julien Jerphanion --- .../tests/src/solver/resolvo/test_solver.cpp | 70 +++++++++---------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 6e8802d7ee..82d2096cd5 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -1480,50 +1480,46 @@ TEST_CASE("Test consistency with libsolv (environment creation)") SECTION("scikit-learn explicit") { - std::vector specs_to_install = { - "python[version=\">=3.10,<3.11.0a0\"]", "pip", - "scikit-learn[version=\">=1.0.0,<1.6a0\"]", "numpy[version=\">=1.20.0,<2.0a0\"]", - "scipy[version=\">=1.10.0,<1.15a0\"]", "joblib[version=\">=1.0.1,<2.0a0\"]", - "threadpoolctl[version=\">=2.1.0,<3.6a0\"]", - }; + std::vector specs_to_install = { "scikit-learn==1.6.1=py313h8ef605b_0" }; std::vector known_resolution = { PackageInfo("_libgcc_mutex", "0.1", "conda_forge", 0), - PackageInfo("python_abi", "3.10", "4_cp310", 0), - PackageInfo("ld_impl_linux-64", "2.40", "hf3520f5_7", 0), - PackageInfo("ca-certificates", "2024.7.4", "hbcca054_0", 0), - PackageInfo("libgomp", "14.1.0", "h77fa898_0", 0), PackageInfo("_openmp_mutex", "4.5", "2_gnu", 0), - PackageInfo("libgcc-ng", "14.1.0", "h77fa898_0", 0), - PackageInfo("openssl", "3.3.1", "h4ab18f5_1", 0), - PackageInfo("libxcrypt", "4.4.36", "hd590300_1", 0), - PackageInfo("libzlib", "1.3.1", "h4ab18f5_1", 0), - PackageInfo("libffi", "3.4.2", "h7f98852_5", 0), - PackageInfo("bzip2", "1.0.8", "hd590300_5", 0), - PackageInfo("ncurses", "6.5", "h59595ed_0", 0), - PackageInfo("libstdcxx-ng", "14.1.0", "hc0a3c3a_0", 0), - PackageInfo("libgfortran5", "14.1.0", "hc5f4f2c_0", 0), + PackageInfo("bzip2", "1.0.8", "h4bc722e_7", 0), + PackageInfo("ca-certificates", "2025.1.31", "hbcca054_0", 0), + PackageInfo("joblib", "1.4.2", "pyhd8ed1ab_1", 0), + PackageInfo("ld_impl_linux-64", "2.43", "h712a8e2_2", 0), + PackageInfo("libblas", "3.9.0", "28_h59b9bed_openblas", 0), + PackageInfo("libcblas", "3.9.0", "28_he106b2a_openblas", 0), + PackageInfo("libexpat", "2.6.4", "h5888daf_0", 0), + PackageInfo("libffi", "3.4.6", "h2dba641_0", 0), + PackageInfo("libgcc", "14.2.0", "h77fa898_1", 0), + PackageInfo("libgcc-ng", "14.2.0", "h69a702a_1", 0), + PackageInfo("libgfortran", "14.2.0", "h69a702a_1", 0), + PackageInfo("libgfortran5", "14.2.0", "hd5240d6_1", 0), + PackageInfo("libgomp", "14.2.0", "h77fa898_1", 0), + PackageInfo("liblapack", "3.9.0", "28_h7ac8fdf_openblas", 0), + PackageInfo("liblzma", "5.6.4", "hb9d3cd8_0", 0), + PackageInfo("libmpdec", "4.0.0", "h4bc722e_0", 0), + PackageInfo("libopenblas", "0.3.28", "pthreads_h94d23a6_1", 0), + PackageInfo("libsqlite", "3.48.0", "hee588c1_1", 0), + PackageInfo("libstdcxx", "14.2.0", "hc0a3c3a_1", 0), PackageInfo("libuuid", "2.38.1", "h0b41bf4_0", 0), - PackageInfo("libnsl", "2.0.1", "hd590300_0", 0), - PackageInfo("xz", "5.2.6", "h166bdaf_0", 0), - PackageInfo("tk", "8.6.13", "noxft_h4845f30_101", 0), - PackageInfo("libsqlite", "3.46.0", "hde9e2c9_0", 0), + PackageInfo("libzlib", "1.3.1", "hb9d3cd8_2", 0), + PackageInfo("ncurses", "6.5", "h2d0b736_3", 0), + PackageInfo("numpy", "2.2.3", "py313h17eae1a_0", 0), + PackageInfo("openssl", "3.4.1", "h7b32b05_0", 0), + // Omitted as added by the environment creation + PackageInfo("pip", "25.0.1", "pyh145f28c_0", 0), + PackageInfo("python", "3.13.2", "hf636f53_100_cp313", 0), + PackageInfo("python_abi", "3.13", "5_cp313", 0), PackageInfo("readline", "8.2", "h8228510_1", 0), - PackageInfo("libgfortran-ng", "14.1.0", "h69a702a_0", 0), - PackageInfo("libopenblas", "0.3.27", "pthreads_hac2b453_1", 0), - PackageInfo("libblas", "3.9.0", "22_linux64_openblas", 0), - PackageInfo("libcblas", "3.9.0", "22_linux64_openblas", 0), - PackageInfo("liblapack", "3.9.0", "22_linux64_openblas", 0), - PackageInfo("tzdata", "2024a", "h0c530f3_0", 0), - PackageInfo("python", "3.10.14", "hd12c33a_0_cpython", 0), - PackageInfo("wheel", "0.43.0", "pyhd8ed1ab_1", 0), - PackageInfo("setuptools", "70.1.1", "pyhd8ed1ab_0", 0), - PackageInfo("pip", "24.0", "pyhd8ed1ab_0", 0), + PackageInfo("scikit-learn", "1.6.1", "py313h8ef605b_0", 0), + PackageInfo("scipy", "1.15.1", "py313h750cbce_0", 0), + PackageInfo("setuptools", "75.8.0", "pyhff2d567_0", 0), PackageInfo("threadpoolctl", "3.5.0", "pyhc1e730c_0", 0), - PackageInfo("joblib", "1.4.2", "pyhd8ed1ab_0", 0), - PackageInfo("numpy", "1.26.4", "py310hb13e2d6_0", 0), - PackageInfo("scipy", "1.14.0", "py310h93e2701_1", 0), - PackageInfo("scikit-learn", "1.5.0", "py310h981052a_1", 1) + PackageInfo("tk", "8.6.13", "noxft_h4845f30_101", 0), + PackageInfo("tzdata", "2025a", "h78e105d_0", 0), }; std::sort( From 14a20107a22db2c4096214171ccfe06567891468 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 18 Feb 2025 09:17:21 +0100 Subject: [PATCH 28/59] Add comment regarding pip Signed-off-by: Julien Jerphanion --- libmamba/tests/src/solver/resolvo/test_solver.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 82d2096cd5..82874ac3f1 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -1480,7 +1480,9 @@ TEST_CASE("Test consistency with libsolv (environment creation)") SECTION("scikit-learn explicit") { - std::vector specs_to_install = { "scikit-learn==1.6.1=py313h8ef605b_0" }; + // Note: currently, pip is added to the environment when python is added + // we add it here to make resolvo's results consistent with libsolv's. + std::vector specs_to_install = { "scikit-learn==1.6.1=py313h8ef605b_0", "pip" }; std::vector known_resolution = { PackageInfo("_libgcc_mutex", "0.1", "conda_forge", 0), From 48147955e879f823081cf00e90f8a4679f2b8bdb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 18 Feb 2025 09:25:22 +0100 Subject: [PATCH 29/59] Reformat with recent prettier pre-commit setup Signed-off-by: Julien Jerphanion --- .../tests/src/solver/libsolv/test_solver.cpp | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/libmamba/tests/src/solver/libsolv/test_solver.cpp b/libmamba/tests/src/solver/libsolv/test_solver.cpp index 8d40905c94..1e8c41cf7d 100644 --- a/libmamba/tests/src/solver/libsolv/test_solver.cpp +++ b/libmamba/tests/src/solver/libsolv/test_solver.cpp @@ -63,41 +63,43 @@ find_actions_with_name(const Solution& solution, std::string_view name) return out; } -auto find_actions(const Solution& solution) -> std::vector +auto +find_actions(const Solution& solution) -> std::vector { auto out = std::vector(); for (const auto& action : solution.actions) { std::visit( - [&](const auto& act) - { - using Act = std::decay_t; - if constexpr (Solution::has_install_v) + [&](const auto& act) { - out.push_back(act); - } - }, - action + using Act = std::decay_t; + if constexpr (Solution::has_install_v) + { + out.push_back(act); + } + }, + action ); } return out; } -auto extract_package_to_install(const Solution& solution) -> std::vector +auto +extract_package_to_install(const Solution& solution) -> std::vector { auto out = std::vector(); for (const auto& action : find_actions(solution)) { std::visit( - [&](const auto& act) - { - using Act = std::decay_t; - if constexpr (Solution::has_install_v) + [&](const auto& act) { - out.push_back(act.install); - } - }, - action + using Act = std::decay_t; + if constexpr (Solution::has_install_v) + { + out.push_back(act.install); + } + }, + action ); } return out; From 83d85c531138f1ec0f38a21e719aa769286f0db5 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 18 Feb 2025 10:38:09 +0100 Subject: [PATCH 30/59] Do not promote warnings as errors Signed-off-by: Julien Jerphanion --- .github/workflows/unix_impl.yml | 2 +- libmambapy/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unix_impl.yml b/.github/workflows/unix_impl.yml index 59acd455fe..c721733c37 100644 --- a/.github/workflows/unix_impl.yml +++ b/.github/workflows/unix_impl.yml @@ -40,7 +40,7 @@ jobs: --preset mamba-unix-shared-${{ inputs.build_type }} \ -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \ -D CMAKE_C_COMPILER_LAUNCHER=sccache \ - -D MAMBA_WARNING_AS_ERROR=ON \ + -D MAMBA_WARNING_AS_ERROR=OFF \ -D BUILD_LIBMAMBAPY=OFF \ -D ENABLE_MAMBA_ROOT_PREFIX_FALLBACK=OFF cmake --build build/ --parallel diff --git a/libmambapy/setup.py b/libmambapy/setup.py index b8bc28281e..3eae12f478 100644 --- a/libmambapy/setup.py +++ b/libmambapy/setup.py @@ -27,7 +27,7 @@ def libmambapy_version(): def get_cmake_args(): cmake_args = [f"-DMAMBA_INSTALL_PYTHON_EXT_LIBDIR={CMAKE_INSTALL_DIR()}/src/libmambapy"] if sys.platform != "win32" and sys.platform != "cygwin": - cmake_args += ["-DMAMBA_WARNING_AS_ERROR=ON"] + cmake_args += ["-DMAMBA_WARNING_AS_ERROR=OFF"] return cmake_args From fd80f657c43f6484c2686e92fc4e4c954a2a5d67 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 18 Feb 2025 13:40:14 +0100 Subject: [PATCH 31/59] Add `resolvo-cpp` as a host dependency for micromamba builds Signed-off-by: Julien Jerphanion --- .github/workflows/static_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/static_build.yml b/.github/workflows/static_build.yml index 7a2b3ce13b..045e29af52 100644 --- a/.github/workflows/static_build.yml +++ b/.github/workflows/static_build.yml @@ -70,6 +70,8 @@ jobs: # Special values for running the feedstock with a local source export FEEDSTOCK_ROOT="${PWD}" export CI="local" + # Patch: add resolvo-cpp as a host dependency + sed -i 's/ - fmt/ - fmt\n - resolvo-cpp/' recipe/meta.yaml # For OSX not using Docker export CONDA_BLD_PATH="${PWD}/build_artifacts" mkdir -p "${CONDA_BLD_PATH}" From 623b3ff1dad271272e05c775f5e4d4202b4adb2e Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 29 Apr 2025 17:57:06 +0200 Subject: [PATCH 32/59] Adapt Database and related signatures Signed-off-by: Julien Jerphanion --- libmamba/CMakeLists.txt | 6 + libmamba/include/mamba/api/channel_loader.hpp | 8 +- libmamba/include/mamba/core/context.hpp | 2 + .../mamba/core/package_database_loader.hpp | 12 +- libmamba/include/mamba/solver/database.hpp | 42 ++ .../include/mamba/solver/resolvo/database.hpp | 262 +++++++ .../include/mamba/solver/resolvo/solver.hpp | 32 + .../include/mamba/solver/solver_factory.hpp | 39 + libmamba/src/api/channel_loader.cpp | 102 ++- libmamba/src/api/configuration.cpp | 11 + libmamba/src/api/install.cpp | 173 +++-- libmamba/src/api/remove.cpp | 9 +- libmamba/src/api/repoquery.cpp | 26 +- libmamba/src/api/update.cpp | 29 +- libmamba/src/core/package_database_loader.cpp | 41 +- libmamba/src/solver/resolvo/database.cpp | 676 ++++++++++++++++++ libmamba/src/solver/resolvo/solver.cpp | 167 +++++ .../libmambapy/bindings/expected_caster.hpp | 123 ++++ libmambapy/src/libmambapy/bindings/solver.cpp | 1 + micromamba/src/update.cpp | 9 +- 20 files changed, 1622 insertions(+), 148 deletions(-) create mode 100644 libmamba/include/mamba/solver/database.hpp create mode 100644 libmamba/include/mamba/solver/resolvo/database.hpp create mode 100644 libmamba/include/mamba/solver/resolvo/solver.hpp create mode 100644 libmamba/include/mamba/solver/solver_factory.hpp create mode 100644 libmamba/src/solver/resolvo/database.cpp create mode 100644 libmamba/src/solver/resolvo/solver.cpp diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index bc52eb6878..fcc66ac215 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -196,6 +196,9 @@ set( ${LIBMAMBA_SOURCE_DIR}/solver/libsolv/repo_info.cpp ${LIBMAMBA_SOURCE_DIR}/solver/libsolv/solver.cpp ${LIBMAMBA_SOURCE_DIR}/solver/libsolv/unsolvable.cpp + # Solver resolvo implementation + ${LIBMAMBA_SOURCE_DIR}/solver/resolvo/database.cpp + ${LIBMAMBA_SOURCE_DIR}/solver/resolvo/solver.cpp # Artifacts validation ${LIBMAMBA_SOURCE_DIR}/validation/errors.cpp ${LIBMAMBA_SOURCE_DIR}/validation/keys.cpp @@ -350,6 +353,9 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/repo_info.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/solver.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/unsolvable.hpp + # Solver resolvo implementation + ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/resolvo/database.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/resolvo/solver.hpp # Artifacts validation ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/errors.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/keys.hpp diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index 3a74e60783..fa9cbe2636 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -8,6 +8,7 @@ #define MAMBA_API_CHANNEL_LOADER_HPP #include "mamba/core/error_handling.hpp" +#include "mamba/solver/resolvo/database.hpp" namespace mamba { @@ -15,6 +16,11 @@ namespace mamba { class Database; } + + namespace solver::resolvo + { + class Database; + } class Context; class ChannelContext; class MultiPackageCache; @@ -30,7 +36,7 @@ namespace mamba auto load_channels( Context& ctx, ChannelContext& channel_context, - solver::libsolv::Database& database, + std::variant& database, MultiPackageCache& package_caches ) -> expected_t; diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index fd65a64549..0b13d88851 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -264,6 +264,8 @@ namespace mamba bool repodata_use_zst = true; std::vector repodata_has_zst = { "https://conda.anaconda.org/conda-forge" }; + bool use_resolvo_solver = false; + // FIXME: Should not be stored here // Notice that we cannot build this map directly from mirrored_channels, // since we need to add a single "mirror" for non mirrored channels diff --git a/libmamba/include/mamba/core/package_database_loader.hpp b/libmamba/include/mamba/core/package_database_loader.hpp index d1df00ecdb..50fe26e60f 100644 --- a/libmamba/include/mamba/core/package_database_loader.hpp +++ b/libmamba/include/mamba/core/package_database_loader.hpp @@ -9,6 +9,7 @@ #include "mamba/core/error_handling.hpp" #include "mamba/solver/libsolv/repo_info.hpp" +#include "mamba/solver/resolvo/database.hpp" #include "mamba/specs/channel.hpp" namespace mamba @@ -22,6 +23,11 @@ namespace mamba class Database; } + namespace solver::resolvo + { + class Database; + } + void add_spdlog_logger_to_database(solver::libsolv::Database& database); auto load_subdir_in_database( // @@ -32,8 +38,10 @@ namespace mamba auto load_installed_packages_in_database( const Context& ctx, - solver::libsolv::Database& database, + std::variant< + std::reference_wrapper, + std::reference_wrapper> database, const PrefixData& prefix - ) -> solver::libsolv::RepoInfo; + ) -> expected_t; } #endif diff --git a/libmamba/include/mamba/solver/database.hpp b/libmamba/include/mamba/solver/database.hpp new file mode 100644 index 0000000000..7037f064e4 --- /dev/null +++ b/libmamba/include/mamba/solver/database.hpp @@ -0,0 +1,42 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_SOLVER_DATABASE_HPP +#define MAMBA_SOLVER_DATABASE_HPP + +#include +#include + +#include "mamba/fs/filesystem.hpp" +#include "mamba/specs/channel.hpp" +#include "mamba/specs/package_info.hpp" + +namespace mamba::solver +{ + class Database + { + public: + + virtual ~Database() = default; + + virtual void add_repo_from_repodata_json( + const fs::u8path& filename, + const std::string& repo_url, + const std::string& channel_id, + bool verify_artifacts = false + ) = 0; + + virtual void add_repo_from_packages( + const std::vector& packages, + const std::string& repo_name, + bool pip_as_python_dependency = false + ) = 0; + + virtual void set_installed_repo(const std::string& repo_name) = 0; + }; +} + +#endif // MAMBA_SOLVER_DATABASE_HPP diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp new file mode 100644 index 0000000000..bc20fec4ba --- /dev/null +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -0,0 +1,262 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_SOLVER_RESOLVO_DATABASE_HPP +#define MAMBA_SOLVER_RESOLVO_DATABASE_HPP + +#include +#include +#include + +#include +#include +#include + +#include "mamba/solver/database.hpp" +#include "mamba/specs/match_spec.hpp" +#include "mamba/specs/package_info.hpp" +#include "mamba/specs/version.hpp" + +namespace std +{ + template <> + struct hash<::resolvo::NameId> + { + size_t operator()(const ::resolvo::NameId& id) const noexcept + { + return static_cast(id.id); + } + }; + + template <> + struct hash<::resolvo::VersionSetId> + { + size_t operator()(const ::resolvo::VersionSetId& id) const noexcept + { + return static_cast(id.id); + } + }; + + template <> + struct hash<::resolvo::SolvableId> + { + size_t operator()(const ::resolvo::SolvableId& id) const noexcept + { + return static_cast(id.id); + } + }; + + template <> + struct hash<::resolvo::StringId> + { + size_t operator()(const ::resolvo::StringId& id) const noexcept + { + return static_cast(id.id); + } + }; +} + +namespace mamba::solver::resolvo +{ + // Create a template Pool class that maps a key to a set of values + template + struct Mapping + { + /** + * Adds the value to the Mapping and returns its associated id. If the + * value is already in the Mapping, returns the id associated with it. + */ + ID alloc(T value) + { + if (auto element = value_to_id.find(value); element != value_to_id.end()) + { + return element->second; + } + auto id = ID{ static_cast(id_to_value.size()) }; + id_to_value[id] = value; + value_to_id[value] = id; + return id; + } + + /** + * Returns the value associated with the given id. + */ + T operator[](ID id) + { + return id_to_value[id]; + } + + /** + * Returns the id associated with the given value. + */ + ID operator[](T value) + { + return value_to_id[value]; + } + + // Iterator for the Mapping + auto begin() + { + return id_to_value.begin(); + } + + auto end() + { + return id_to_value.end(); + } + + auto begin() const + { + return id_to_value.begin(); + } + + auto end() const + { + return id_to_value.end(); + } + + auto cbegin() + { + return id_to_value.cbegin(); + } + + auto cend() + { + return id_to_value.cend(); + } + + auto cbegin() const + { + return id_to_value.cbegin(); + } + + auto cend() const + { + return id_to_value.cend(); + } + + auto find(T value) + { + return value_to_id.find(value); + } + + auto begin_ids() + { + return value_to_id.begin(); + } + + auto end_ids() + { + return value_to_id.end(); + } + + auto begin_ids() const + { + return value_to_id.begin(); + } + + auto end_ids() const + { + return value_to_id.end(); + } + + auto cbegin_ids() + { + return value_to_id.cbegin(); + } + + auto cend_ids() + { + return value_to_id.cend(); + } + + auto cbegin_ids() const + { + return value_to_id.cbegin(); + } + + auto cend_ids() const + { + return value_to_id.cend(); + } + + auto size() const + { + return id_to_value.size(); + } + + private: + + std::unordered_map value_to_id; + std::unordered_map id_to_value; + }; + + class Database final + : public mamba::solver::Database + , public ::resolvo::DependencyProvider + { + public: + + Database(); + ~Database() override = default; + + // Implementation of mamba::solver::Database interface + void add_repo_from_repodata_json( + const fs::u8path& filename, + const std::string& repo_url, + const std::string& channel_id, + bool verify_artifacts = false + ) override; + + void add_repo_from_packages( + const std::vector& packages, + const std::string& repo_name, + bool pip_as_python_dependency = false + ) override; + + void set_installed_repo(const std::string& repo_name) override; + + // Implementation of resolvo::DependencyProvider interface + ::resolvo::String display_solvable(::resolvo::SolvableId solvable) override; + ::resolvo::String display_solvable_name(::resolvo::SolvableId solvable) override; + ::resolvo::String + display_merged_solvables(::resolvo::Slice<::resolvo::SolvableId> solvable) override; + ::resolvo::String display_name(::resolvo::NameId name) override; + ::resolvo::String display_version_set(::resolvo::VersionSetId version_set) override; + ::resolvo::String display_string(::resolvo::StringId string) override; + ::resolvo::NameId version_set_name(::resolvo::VersionSetId version_set_id) override; + ::resolvo::NameId solvable_name(::resolvo::SolvableId solvable_id) override; + ::resolvo::Candidates get_candidates(::resolvo::NameId package) override; + void sort_candidates(::resolvo::Slice<::resolvo::SolvableId> solvables) override; + ::resolvo::Vector<::resolvo::SolvableId> filter_candidates( + ::resolvo::Slice<::resolvo::SolvableId> candidates, + ::resolvo::VersionSetId version_set_id, + bool inverse + ) override; + ::resolvo::Dependencies get_dependencies(::resolvo::SolvableId solvable_id) override; + + // Public access to pools and helper methods + ::resolvo::VersionSetId alloc_version_set(std::string_view raw_match_spec); + ::resolvo::SolvableId alloc_solvable(specs::PackageInfo package_info); + std::pair + find_highest_version(::resolvo::VersionSetId version_set_id); + + // Pools for mapping between resolvo IDs and mamba types + Mapping<::resolvo::NameId, ::resolvo::String> name_pool; + Mapping<::resolvo::StringId, ::resolvo::String> string_pool; + Mapping<::resolvo::VersionSetId, specs::MatchSpec> version_set_pool; + Mapping<::resolvo::SolvableId, specs::PackageInfo> solvable_pool; + + private: + + // Maps for quick lookups + std::unordered_map<::resolvo::NameId, ::resolvo::Vector<::resolvo::SolvableId>> name_to_solvable; + std::unordered_map<::resolvo::VersionSetId, std::pair> + version_set_to_max_version_and_track_features_numbers; + }; +} + +#endif // MAMBA_SOLVER_RESOLVO_DATABASE_HPP diff --git a/libmamba/include/mamba/solver/resolvo/solver.hpp b/libmamba/include/mamba/solver/resolvo/solver.hpp new file mode 100644 index 0000000000..9f17f08322 --- /dev/null +++ b/libmamba/include/mamba/solver/resolvo/solver.hpp @@ -0,0 +1,32 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_SOLVER_RESOLVO_SOLVER_HPP +#define MAMBA_SOLVER_RESOLVO_SOLVER_HPP + +#include "mamba/core/error_handling.hpp" +#include "mamba/solver/request.hpp" +#include "mamba/solver/solution.hpp" + +namespace mamba::solver::resolvo +{ + class Database; + + class Solver + { + public: + + using Outcome = std::variant; + + [[nodiscard]] auto solve(Database& database, Request&& request) -> expected_t; + [[nodiscard]] auto solve(Database& database, const Request& request) -> expected_t; + + private: + + auto solve_impl(Database& database, const Request& request) -> expected_t; + }; +} +#endif diff --git a/libmamba/include/mamba/solver/solver_factory.hpp b/libmamba/include/mamba/solver/solver_factory.hpp new file mode 100644 index 0000000000..37a4da4dd5 --- /dev/null +++ b/libmamba/include/mamba/solver/solver_factory.hpp @@ -0,0 +1,39 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_SOLVER_SOLVER_FACTORY_HPP +#define MAMBA_SOLVER_SOLVER_FACTORY_HPP + +#include + +#include "mamba/core/context.hpp" +#include "mamba/solver/libsolv/database.hpp" +#include "mamba/solver/libsolv/solver.hpp" +#include "mamba/solver/resolvo/database.hpp" +#include "mamba/solver/resolvo/solver.hpp" + +namespace mamba::solver +{ + /** + * Create a solver based on the configuration. + * + * @param ctx The context containing the configuration. + * @return A unique pointer to the appropriate solver. + */ + template + auto create_solver(const Context& ctx) + { + if (ctx.use_resolvo_solver) + { + return std::make_unique(); + } + else + { + return std::make_unique(); + } + } +} +#endif diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 1b442409de..2abb7c79a4 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -11,6 +11,7 @@ #include "mamba/core/package_database_loader.hpp" #include "mamba/core/prefix_data.hpp" #include "mamba/core/subdir_index.hpp" +#include "mamba/core/virtual_packages.hpp" #include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/libsolv/repo_info.hpp" #include "mamba/specs/package_info.hpp" @@ -46,7 +47,24 @@ namespace mamba } prefix_data.load_single_record(repodata_record_json); } - return load_installed_packages_in_database(ctx, database, prefix_data); + + // Create a repo from the packages + auto repo = database.add_repo_from_packages( + prefix_data.sorted_records(), + "pkgs_dir", + solver::libsolv::PipAsPythonDependency::No + ); + + // Load the packages into the database + load_installed_packages_in_database( + ctx, + std::variant< + std::reference_wrapper, + std::reference_wrapper>(std::ref(database)), + prefix_data + ); + + return repo; } void create_subdirs( @@ -131,7 +149,7 @@ namespace mamba auto load_channels_impl( Context& ctx, ChannelContext& channel_context, - solver::libsolv::Database& database, + std::variant& database, MultiPackageCache& package_caches, bool is_retry ) -> expected_t @@ -202,7 +220,16 @@ namespace mamba if (!packages.empty()) { - database.add_repo_from_packages(packages, "packages"); + if (auto* libsolv_db = std::get_if(&database)) + { + libsolv_db->add_repo_from_packages(packages, "packages"); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + (void) resolvo_db; // Silence unused variable warning + // TODO: Implement this for resolvo + throw std::runtime_error("Offline mode not supported with resolvo solver yet"); + } } expected_t download_res; @@ -249,7 +276,16 @@ namespace mamba LOG_INFO << "Creating repo from pkgs_dir for offline"; for (const auto& c : ctx.pkgs_dirs) { - create_repo_from_pkgs_dir(ctx, channel_context, database, c); + if (auto* libsolv_db = std::get_if(&database)) + { + create_repo_from_pkgs_dir(ctx, channel_context, *libsolv_db, c); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + (void) resolvo_db; // Silence unused variable warning + // TODO: Implement this for resolvo + throw std::runtime_error("Offline mode not supported with resolvo solver yet"); + } } } std::string prev_channel; @@ -269,31 +305,41 @@ namespace mamba continue; } - load_subdir_in_database(ctx, database, subdir) - .transform([&](solver::libsolv::RepoInfo&& repo) - { database.set_repo_priority(repo, priorities[i]); }) - .or_else( - [&](const auto&) - { - if (is_retry) - { - std::stringstream ss; - ss << "Could not load repodata.json for " << subdir.name() - << " after retry." << "Please check repodata source. Exiting." - << std::endl; - error_list.push_back( - mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) - ); - } - else + if (auto* libsolv_db = std::get_if(&database)) + { + load_subdir_in_database(ctx, *libsolv_db, subdir) + .transform([&](solver::libsolv::RepoInfo&& repo) + { libsolv_db->set_repo_priority(repo, priorities[i]); }) + .or_else( + [&](const auto&) { - LOG_WARNING << "Could not load repodata.json for " << subdir.name() - << ". Deleting cache, and retrying."; - subdir.clear_valid_cache_files(); - loading_failed = true; + if (is_retry) + { + std::stringstream ss; + ss << "Could not load repodata.json for " << subdir.name() + << " after retry." + << "Please check repodata source. Exiting." << std::endl; + error_list.push_back( + mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) + ); + } + else + { + LOG_WARNING << "Could not load repodata.json for " + << subdir.name() + << ". Deleting cache, and retrying."; + subdir.clear_valid_cache_files(); + loading_failed = true; + } } - } - ); + ); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + (void) resolvo_db; // Silence unused variable warning + // TODO: Implement this for resolvo + throw std::runtime_error("Loading subdirs not supported with resolvo solver yet"); + } } if (loading_failed) @@ -318,7 +364,7 @@ namespace mamba auto load_channels( Context& ctx, ChannelContext& channel_context, - solver::libsolv::Database& database, + std::variant& database, MultiPackageCache& package_caches ) -> expected_t { diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index e4429fea4c..611e16fe26 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -1609,6 +1609,17 @@ namespace mamba } )); + insert(Configurable("experimental_resolvo_solver", false) + .group("Solver") + .set_rc_configurable() + .set_env_var_names() + .description("Use the experimental resolvo solver instead of libsolv") + .long_description(unindent(R"( + When enabled, use the experimental resolvo solver instead of libsolv. + This is an experimental feature and may not be fully functional.)")) + .set_post_merge_hook([&](bool& value) + { m_context.use_resolvo_solver = value; })); + insert(Configurable("explicit_install", false) .group("Solver") .description("Use explicit install instead of solving environment")); diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 8f6dcad452..b654acc4ad 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include "mamba/api/channel_loader.hpp" @@ -25,6 +26,7 @@ #include "mamba/download/downloader.hpp" #include "mamba/fs/filesystem.hpp" #include "mamba/solver/libsolv/solver.hpp" +#include "mamba/solver/resolvo/solver.hpp" #include "mamba/util/path_manip.hpp" #include "mamba/util/string.hpp" @@ -552,16 +554,24 @@ namespace mamba LOG_WARNING << "No 'channels' specified"; } - solver::libsolv::Database db{ + const std::vector matchspec_parsers{ + ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv + }; + solver::libsolv::Database libsolv_db{ channel_context.params(), { ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv, }, }; - add_spdlog_logger_to_database(db); + add_spdlog_logger_to_database(libsolv_db); - auto maybe_load = load_channels(ctx, channel_context, db, package_caches); + std::variant db_variant( + std::in_place_type, + std::move(libsolv_db) + ); + auto maybe_load = load_channels(ctx, channel_context, db_variant, package_caches); if (!maybe_load) { throw std::runtime_error(maybe_load.error().what()); @@ -574,11 +584,25 @@ namespace mamba } PrefixData& prefix_data = maybe_prefix_data.value(); - load_installed_packages_in_database(ctx, db, prefix_data); - + if (auto* libsolv_db = std::get_if(&db_variant)) + { + load_installed_packages_in_database(ctx, *libsolv_db, prefix_data); + } + else if (auto* resolvo_db = std::get_if(&db_variant)) + { + load_installed_packages_in_database(ctx, *resolvo_db, prefix_data); + } auto request = create_install_request(prefix_data, raw_specs, freeze_installed); - add_pins_to_request(request, ctx, prefix_data, raw_specs, no_pin, no_py_pin); + add_pins_to_request( + request, + ctx, + prefix_data, + raw_specs, + /* no_pin= */ config.at("no_pin").value(), + /* no_py_pin = */ config.at("no_py_pin").value() + ); + request.flags = ctx.solver_flags; { @@ -587,57 +611,49 @@ namespace mamba // Console stream prints on destruction } - auto outcome = solver::libsolv::Solver() - .solve( - db, - request, - ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Mixed - ) - .value(); + auto outcome = solver::libsolv::Solver().solve( + std::get(db_variant), + request, + ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv + ); - if (auto* unsolvable = std::get_if(&outcome)) + if (!outcome.has_value()) + { + throw std::runtime_error(outcome.error().what()); + } + auto& result = outcome.value(); + if (auto* unsolvable = std::get_if(&result)) { unsolvable->explain_problems_to( - db, - LOG_ERROR, - { - /* .unavailable= */ ctx.graphics_params.palette.failure, - /* .available= */ ctx.graphics_params.palette.success, - } + std::get(db_variant), + std::cout, + mamba::solver::ProblemsMessageFormat{} ); - if (retry_clean_cache && !is_retry) - { - ctx.local_repodata_ttl = 2; - bool retry = true; - return install_specs_impl( - ctx, - channel_context, - config, - raw_specs, - create_env, - remove_prefix_on_failure, - retry - ); - } - if (freeze_installed) - { - Console::instance().print("Possible hints:\n - 'freeze_installed' is turned on\n" - ); - } - if (ctx.output_params.json) { - Console::instance().json_write( - { { "success", false }, { "solver_problems", unsolvable->problems(db) } } + nlohmann::json j; + j["success"] = false; + j["solver_problems"] = unsolvable->problems( + std::get(db_variant) ); + Console::instance().json_write(j); } throw mamba_error( "Could not solve for environment specs", mamba_error_code::satisfiablitity_error ); } + auto& solution = std::get(result); + + Console::instance().json_write({ { "success", true } }); + auto transaction = MTransaction( + ctx, + std::get(db_variant), + request, + solution, + package_caches + ); std::vector locks; @@ -646,42 +662,21 @@ namespace mamba locks.push_back(LockFile(c)); } - Console::instance().json_write({ { "success", true } }); - - // The point here is to delete the database before executing the transaction. - // The database can have high memory impact, since installing packages - // requires downloading, extracting, and launching Python interpreters for - // creating ``.pyc`` files. - // Ideally this whole function should be properly refactored and the transaction itself - // should not need the database. - auto trans = [&](auto database) - { - return MTransaction( // - ctx, - database, - request, - std::get(outcome), - package_caches - ); - }(std::move(db)); - if (ctx.output_params.json) { - trans.log_json(); + transaction.log_json(); } Console::stream(); - if (trans.prompt(ctx, channel_context)) + if (transaction.prompt(ctx, channel_context)) { if (create_env && !ctx.dry_run) { detail::create_target_directory(ctx, ctx.prefix_params.target_prefix); } - detail::populate_state_file(ctx.prefix_params.target_prefix, env_vars, no_env); - - trans.execute(ctx, channel_context, prefix_data); + transaction.execute(ctx, channel_context, prefix_data); // Print activation message only if the environment is freshly created if (create_env) @@ -746,14 +741,27 @@ namespace mamba bool remove_prefix_on_failure ) { - solver::libsolv::Database database{ - channel_context.params(), - { - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - }, + const std::vector matchspec_parsers2{ + ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv }; - add_spdlog_logger_to_database(database); + std::variant + db = ctx.use_resolvo_solver + ? std::variant< + solver::libsolv::Database, + solver::resolvo::Database>(std::in_place_type) + : std::variant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); + if (!ctx.use_resolvo_solver) + { + add_spdlog_logger_to_database(std::get(db)); + } init_channels(ctx, channel_context); // Some use cases provide a list of explicit specs, but an empty @@ -772,12 +780,23 @@ namespace mamba MultiPackageCache pkg_caches(ctx.pkgs_dirs, ctx.validation_params); - load_installed_packages_in_database(ctx, database, prefix_data); + if (auto* libsolv_db = std::get_if(&db)) + { + load_installed_packages_in_database(ctx, *libsolv_db, prefix_data); + } + else if (auto* resolvo_db = std::get_if(&db)) + { + load_installed_packages_in_database(ctx, *resolvo_db, prefix_data); + } std::vector others; // Note that the Transaction will gather the Solvables, // so they must have been ready in the database's pool before this line - auto transaction = create_transaction(database, pkg_caches, others); + auto transaction = create_transaction( + std::get(db), + pkg_caches, + others + ); std::vector lock_pkgs; diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index 21bc8cbdbc..5697984a84 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -144,7 +144,14 @@ namespace mamba }, }; add_spdlog_logger_to_database(database); - load_installed_packages_in_database(ctx, database, prefix_data); + + load_installed_packages_in_database( + ctx, + std::variant< + std::reference_wrapper, + std::reference_wrapper>(std::ref(database)), + prefix_data + ); const fs::u8path pkgs_dirs(ctx.prefix_params.root_prefix / "pkgs"); MultiPackageCache package_caches({ pkgs_dirs }, ctx.validation_params); diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index d2e4189621..0096e5ddfa 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include "mamba/api/channel_loader.hpp" #include "mamba/api/configuration.hpp" @@ -23,6 +24,7 @@ namespace mamba { auto repoquery_init(Context& ctx, Configuration& config, QueryResultFormat format, bool use_local) + -> solver::libsolv::Database& { config.at("use_target_prefix_fallback").set_value(true); config.at("use_default_prefix_fallback").set_value(true); @@ -34,14 +36,14 @@ namespace mamba config.load(); auto channel_context = ChannelContext::make_conda_compatible(ctx); - solver::libsolv::Database db{ + static std::variant db( + std::in_place_type, channel_context.params(), - { - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - }, - }; - add_spdlog_logger_to_database(db); + solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); + add_spdlog_logger_to_database(std::get(db)); // bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds); MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); @@ -62,7 +64,11 @@ namespace mamba } PrefixData& prefix_data = exp_prefix_data.value(); - load_installed_packages_in_database(ctx, db, prefix_data); + load_installed_packages_in_database( + ctx, + std::get(db), + prefix_data + ); if (format != QueryResultFormat::Json) { @@ -83,7 +89,7 @@ namespace mamba throw std::runtime_error(exp_load.error().what()); } } - return db; + return std::get(db); } } @@ -189,7 +195,7 @@ namespace mamba ) { auto& ctx = config.context(); - auto db = repoquery_init(ctx, config, format, use_local); + auto& db = repoquery_init(ctx, config, format, use_local); return make_repoquery( db, type, diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index f3dd4b44e0..1068470438 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -156,14 +156,14 @@ namespace mamba populate_context_channels_from_specs(raw_update_specs, ctx); - solver::libsolv::Database db{ + std::variant db( + std::in_place_type, channel_context.params(), - { - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - }, - }; - add_spdlog_logger_to_database(db); + solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); + add_spdlog_logger_to_database(std::get(db)); MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); @@ -181,7 +181,7 @@ namespace mamba } PrefixData& prefix_data = exp_prefix_data.value(); - load_installed_packages_in_database(ctx, db, prefix_data); + load_installed_packages_in_database(ctx, std::get(db), prefix_data); auto request = create_update_request(prefix_data, raw_update_specs, update_params); add_pins_to_request( @@ -203,7 +203,7 @@ namespace mamba auto outcome = solver::libsolv::Solver() .solve( - db, + std::get(db), request, ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba @@ -213,7 +213,7 @@ namespace mamba if (auto* unsolvable = std::get_if(&outcome)) { unsolvable->explain_problems_to( - db, + std::get(db), LOG_ERROR, { /* .unavailable= */ ctx.graphics_params.palette.failure, @@ -222,8 +222,11 @@ namespace mamba ); if (ctx.output_params.json) { - Console::instance().json_write({ { "success", false }, - { "solver_problems", unsolvable->problems(db) } }); + Console::instance().json_write( + { { "success", false }, + { "solver_problems", + unsolvable->problems(std::get(db)) } } + ); } throw mamba_error( "Could not solve for environment specs", @@ -234,7 +237,7 @@ namespace mamba Console::instance().json_write({ { "success", true } }); auto transaction = MTransaction( ctx, - db, + std::get(db), request, std::get(outcome), package_caches diff --git a/libmamba/src/core/package_database_loader.cpp b/libmamba/src/core/package_database_loader.cpp index 6c03acf3bd..581e378194 100644 --- a/libmamba/src/core/package_database_loader.cpp +++ b/libmamba/src/core/package_database_loader.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include @@ -140,26 +141,40 @@ namespace mamba auto load_installed_packages_in_database( const Context& ctx, - solver::libsolv::Database& database, + std::variant< + std::reference_wrapper, + std::reference_wrapper> database, const PrefixData& prefix - ) -> solver::libsolv::RepoInfo + ) -> expected_t { - // TODO(C++20): We could do a PrefixData range that returns packages without storing them. auto pkgs = prefix.sorted_records(); - // TODO(C++20): We only need a range that concatenate both for (auto&& pkg : get_virtual_packages(ctx.platform)) { pkgs.push_back(std::move(pkg)); } - // Not adding Pip dependency since it might needlessly make the installed/active environment - // broken if pip is not already installed (debatable). - auto repo = database.add_repo_from_packages( - pkgs, - "installed", - solver::libsolv::PipAsPythonDependency::No - ); - database.set_installed_repo(repo); - return repo; + if (auto* libsolv_db = std::get_if>(&database + )) + { + auto repo = libsolv_db->get().add_repo_from_packages( + pkgs, + "installed", + solver::libsolv::PipAsPythonDependency::No + ); + libsolv_db->get().set_installed_repo(repo); + return {}; + } + else if (auto* resolvo_db = std::get_if>( + &database + )) + { + resolvo_db->get().add_repo_from_packages(pkgs, "installed"); + resolvo_db->get().set_installed_repo("installed"); + return {}; + } + else + { + return make_unexpected("Unknown database type", mamba_error_code::unknown); + } } } diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp new file mode 100644 index 0000000000..f1756b807e --- /dev/null +++ b/libmamba/src/solver/resolvo/database.cpp @@ -0,0 +1,676 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#include + +#include "mamba/core/output.hpp" +#include "mamba/core/util.hpp" +#include "mamba/solver/libsolv/parameters.hpp" +#include "mamba/solver/resolvo/database.hpp" +#include "mamba/specs/channel.hpp" +#include "mamba/specs/package_info.hpp" +#include "mamba/util/string.hpp" + +namespace mamba::solver::resolvo +{ + namespace + { + // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` + auto lsplit_track_features(std::string_view features) + { + constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); + } + + // TODO: factorise with the implementation from `set_solvable` in + // `mamba/solver/libsolv/helpers.cpp` + bool parse_packageinfo_json( + const std::string_view& filename, + const simdjson::dom::element& pkg, + const specs::CondaURL& repo_url, + const std::string& channel_id, + Database& database + ) + { + specs::PackageInfo package_info; + + package_info.channel = channel_id; + package_info.filename = filename; + package_info.package_url = (repo_url / filename).str(specs::CondaURL::Credentials::Show); + + if (auto fn = pkg["fn"].get_string(); !fn.error()) + { + package_info.name = fn.value_unsafe(); + } + else + { + // Fallback from key entry + package_info.name = filename; + } + + if (auto name = pkg["name"].get_string(); !name.error()) + { + package_info.name = name.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")"; + return false; + } + + if (auto version = pkg["version"].get_string(); !version.error()) + { + package_info.version = version.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")"; + return false; + } + + if (auto build_string = pkg["build"].get_string(); !build_string.error()) + { + package_info.build_string = build_string.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")"; + return false; + } + + if (auto build_number = pkg["build_number"].get_uint64(); !build_number.error()) + { + package_info.build_number = build_number.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")"; + return false; + } + + if (auto subdir = pkg["subdir"].get_c_str(); !subdir.error()) + { + package_info.platform = subdir.value_unsafe(); + } + else + { + LOG_WARNING << R"(Found invalid subdir in ")" << filename << R"(")"; + } + + if (auto size = pkg["size"].get_uint64(); !size.error()) + { + package_info.size = size.value_unsafe(); + } + + if (auto md5 = pkg["md5"].get_c_str(); !md5.error()) + { + package_info.md5 = md5.value_unsafe(); + } + + if (auto sha256 = pkg["sha256"].get_c_str(); !sha256.error()) + { + package_info.sha256 = sha256.value_unsafe(); + } + + if (auto elem = pkg["noarch"]; !elem.error()) + { + // TODO: is the following right? + if (auto val = elem.get_bool(); !val.error() && val.value_unsafe()) + { + package_info.noarch = specs::NoArchType::No; + } + else if (auto noarch = elem.get_c_str(); !noarch.error()) + { + package_info.noarch = specs::NoArchType::No; + } + } + + if (auto license = pkg["license"].get_c_str(); !license.error()) + { + package_info.license = license.value_unsafe(); + } + + // TODO conda timestamp are not Unix timestamp. + // Libsolv normalize them this way, we need to do the same here otherwise the current + // package may get arbitrary priority. + if (auto timestamp = pkg["timestamp"].get_uint64(); !timestamp.error()) + { + const auto time = timestamp.value_unsafe(); + // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` + constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; + package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; + } + + if (auto depends = pkg["depends"].get_array(); !depends.error()) + { + for (auto elem : depends) + { + if (auto dep = elem.get_c_str(); !dep.error()) + { + package_info.dependencies.emplace_back(dep.value_unsafe()); + } + } + } + + if (auto constrains = pkg["constrains"].get_array(); !constrains.error()) + { + for (auto elem : constrains) + { + if (auto cons = elem.get_c_str(); !cons.error()) + { + package_info.constrains.emplace_back(cons.value_unsafe()); + } + } + } + + if (auto obj = pkg["track_features"]; !obj.error()) + { + if (auto track_features_arr = obj.get_array(); !track_features_arr.error()) + { + for (auto elem : track_features_arr) + { + if (auto feat = elem.get_string(); !feat.error()) + { + package_info.track_features.emplace_back(feat.value_unsafe()); + } + } + } + else if (auto track_features_str = obj.get_string(); !track_features_str.error()) + { + auto splits = lsplit_track_features(track_features_str.value_unsafe()); + while (!splits[0].empty()) + { + package_info.track_features.emplace_back(splits[0]); + splits = lsplit_track_features(splits[1]); + } + } + } + + database.alloc_solvable(package_info); + return true; + } + } + + Database::Database() + : name_pool(Mapping<::resolvo::NameId, ::resolvo::String>()) + { + } + + void Database::add_repo_from_repodata_json( + const fs::u8path& filename, + const std::string& repo_url, + const std::string& channel_id, + bool verify_artifacts + ) + { + auto parser = simdjson::dom::parser(); + const auto lock = LockFile(filename); + const auto repodata = parser.load(filename); + + // An override for missing package subdir is found at the top level + auto default_subdir = std::string(); + if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); !subdir.error()) + { + default_subdir = std::string(subdir.value_unsafe()); + } + + // Get `base_url` in case 'repodata_version': 2 + // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md + auto base_url = repo_url; + if (auto repodata_version = repodata["repodata_version"].get_int64(); + !repodata_version.error()) + { + if (repodata_version.value_unsafe() == 2) + { + if (auto url = repodata.at_pointer("/info/base_url").get_string(); !url.error()) + { + base_url = std::string(url.value_unsafe()); + } + } + } + + const auto parsed_url = specs::CondaURL::parse(base_url) + .or_else([](specs::ParseError&& err) { throw std::move(err); }) + .value(); + + auto signatures = std::optional(std::nullopt); + if (auto maybe_sigs = repodata["signatures"].get_object(); + !maybe_sigs.error() && verify_artifacts) + { + signatures = std::move(maybe_sigs).value(); + } + + auto added = util::flat_set(); + if (auto pkgs = repodata["packages.conda"].get_object(); !pkgs.error()) + { + for (auto [key, value] : pkgs.value()) + { + parse_packageinfo_json(key, value, parsed_url, channel_id, *this); + } + } + if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) + { + for (auto [key, value] : pkgs.value()) + { + parse_packageinfo_json(key, value, parsed_url, channel_id, *this); + } + } + } + + void Database::add_repo_from_packages( + const std::vector& packages, + [[maybe_unused]] const std::string& repo_name, + [[maybe_unused]] bool pip_as_python_dependency + ) + { + for (const auto& package : packages) + { + alloc_solvable(package); + } + } + + void Database::set_installed_repo([[maybe_unused]] const std::string& repo_name) + { + // TODO: Implement this + } + + ::resolvo::String Database::display_solvable(::resolvo::SolvableId solvable) + { + const specs::PackageInfo& package_info = solvable_pool[solvable]; + return ::resolvo::String{ package_info.long_str() }; + } + + ::resolvo::String Database::display_solvable_name(::resolvo::SolvableId solvable) + { + const specs::PackageInfo& package_info = solvable_pool[solvable]; + return ::resolvo::String{ package_info.name }; + } + + ::resolvo::String + Database::display_merged_solvables(::resolvo::Slice<::resolvo::SolvableId> solvable) + { + std::string result; + for (auto& solvable_id : solvable) + { + result += solvable_pool[solvable_id].long_str(); + } + return ::resolvo::String{ result }; + } + + ::resolvo::String Database::display_name(::resolvo::NameId name) + { + return name_pool[name]; + } + + ::resolvo::String Database::display_version_set(::resolvo::VersionSetId version_set) + { + const specs::MatchSpec match_spec = version_set_pool[version_set]; + return ::resolvo::String{ match_spec.to_string() }; + } + + ::resolvo::String Database::display_string(::resolvo::StringId string) + { + return string_pool[string]; + } + + ::resolvo::NameId Database::version_set_name(::resolvo::VersionSetId version_set_id) + { + const specs::MatchSpec match_spec = version_set_pool[version_set_id]; + return name_pool[::resolvo::String{ match_spec.name().to_string() }]; + } + + ::resolvo::NameId Database::solvable_name(::resolvo::SolvableId solvable_id) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + return name_pool[::resolvo::String{ package_info.name }]; + } + + ::resolvo::Candidates Database::get_candidates(::resolvo::NameId package) + { + ::resolvo::Candidates candidates{}; + candidates.favored = nullptr; + candidates.locked = nullptr; + candidates.candidates = name_to_solvable[package]; + return candidates; + } + + void Database::sort_candidates(::resolvo::Slice<::resolvo::SolvableId> solvables) + { + std::sort( + solvables.begin(), + solvables.end(), + [&](const ::resolvo::SolvableId& a, const ::resolvo::SolvableId& b) + { + const specs::PackageInfo& package_info_a = solvable_pool[a]; + const specs::PackageInfo& package_info_b = solvable_pool[b]; + + // If track features are present, prefer the solvable having the least of them. + if (package_info_a.track_features.size() != package_info_b.track_features.size()) + { + return package_info_a.track_features.size() + < package_info_b.track_features.size(); + } + + const auto a_version = specs::Version::parse(package_info_a.version).value(); + const auto b_version = specs::Version::parse(package_info_b.version).value(); + + if (a_version != b_version) + { + return a_version > b_version; + } + + if (package_info_a.build_number != package_info_b.build_number) + { + return package_info_a.build_number > package_info_b.build_number; + } + + // Compare the dependencies of the variants. + std::unordered_map<::resolvo::NameId, ::resolvo::VersionSetId> a_deps; + std::unordered_map<::resolvo::NameId, ::resolvo::VersionSetId> b_deps; + for (auto dep_a : package_info_a.dependencies) + { + // TODO: have a VersionID to NameID mapping instead + specs::MatchSpec ms = specs::MatchSpec::parse(dep_a).value(); + const std::string& name = ms.name().to_string(); + auto name_id = name_pool.alloc(::resolvo::String{ name }); + + a_deps[name_id] = version_set_pool[ms]; + } + for (auto dep_b : package_info_b.dependencies) + { + // TODO: have a VersionID to NameID mapping instead + specs::MatchSpec ms = specs::MatchSpec::parse(dep_b).value(); + const std::string& name = ms.name().to_string(); + auto name_id = name_pool.alloc(::resolvo::String{ name }); + + b_deps[name_id] = version_set_pool[ms]; + } + + auto ordering_score = 0; + for (auto [name_id, version_set_id] : a_deps) + { + if (b_deps.find(name_id) != b_deps.end()) + { + auto [a_tf_version, a_n_track_features] = find_highest_version(version_set_id); + auto [b_tf_version, b_n_track_features] = find_highest_version(b_deps[name_id] + ); + + // Favor the solvable with higher versions of their dependencies + if (a_tf_version != b_tf_version) + { + ordering_score += a_tf_version > b_tf_version ? 1 : -1; + } + + // Highly penalize the solvable if a dependencies has more track features + if (a_n_track_features != b_n_track_features) + { + ordering_score += a_n_track_features > b_n_track_features ? -100 : 100; + } + } + } + + if (ordering_score != 0) + { + return ordering_score > 0; + } + + return package_info_a.timestamp > package_info_b.timestamp; + } + ); + } + + ::resolvo::Vector<::resolvo::SolvableId> Database::filter_candidates( + ::resolvo::Slice<::resolvo::SolvableId> candidates, + ::resolvo::VersionSetId version_set_id, + bool inverse + ) + { + specs::MatchSpec match_spec = version_set_pool[version_set_id]; + ::resolvo::Vector<::resolvo::SolvableId> filtered; + + if (inverse) + { + for (auto& solvable_id : candidates) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + + // Is it an appropriate check? Or must another one be crafted? + if (!match_spec.contains_except_channel(package_info)) + { + filtered.push_back(solvable_id); + } + } + } + else + { + for (auto& solvable_id : candidates) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + + // Is it an appropriate check? Or must another one be crafted? + if (match_spec.contains_except_channel(package_info)) + { + filtered.push_back(solvable_id); + } + } + } + + return filtered; + } + + ::resolvo::Dependencies Database::get_dependencies(::resolvo::SolvableId solvable_id) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + + ::resolvo::Dependencies dependencies; + + // TODO: do this in O(1) + for (auto& dep : package_info.dependencies) + { + const specs::MatchSpec match_spec = specs::MatchSpec::parse(dep).value(); + dependencies.requirements.push_back(version_set_pool[match_spec]); + } + for (auto& constr : package_info.constrains) + { + // if constr contain " == " replace it with "==" + std::string constr2 = constr; + while (constr2.find(" == ") != std::string::npos) + { + constr2 = constr2.replace(constr2.find(" == "), 4, "=="); + } + while (constr2.find(" >= ") != std::string::npos) + { + constr2 = constr2.replace(constr2.find(" >= "), 4, ">="); + } + const specs::MatchSpec match_spec = specs::MatchSpec::parse(constr2).value(); + dependencies.constrains.push_back(version_set_pool[match_spec]); + } + + return dependencies; + } + + ::resolvo::VersionSetId Database::alloc_version_set(std::string_view raw_match_spec) + { + std::string raw_match_spec_str = std::string(raw_match_spec); + // Replace all " v" with simply " " to work around the `v` prefix in some version strings + // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in + // `inform2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` while + // (raw_match_spec_str.find(" v") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(" v"), 2, " "); + } + + // Remove any presence of selector on python version in the match spec + // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in + // `infowillow-1.6.3-pyhd8ed1ab_0.conda` + for (const auto specifier : { "=py", "py", ">=py", "<=py", "!=py" }) + { + while (raw_match_spec_str.find(specifier) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(specifier)); + } + } + // Remove any white space between version + // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in + // `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` + while (raw_match_spec_str.find(", ") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(", "), 2, ","); + } + + // TODO: skip allocation for now if "*.*" is in the match spec + if (raw_match_spec_str.find("*.*") != std::string::npos) + { + return ::resolvo::VersionSetId{ 0 }; + } + + // NOTE: works around `openblas 0.2.18|0.2.18.*.` from + // `dlib==19.0=np110py27_blas_openblas_200` If contains "|", split on it and recurse + if (raw_match_spec_str.find("|") != std::string::npos) + { + std::vector match_specs; + std::string match_spec; + for (char c : raw_match_spec_str) + { + if (c == '|') + { + match_specs.push_back(match_spec); + match_spec.clear(); + } + else + { + match_spec += c; + } + } + match_specs.push_back(match_spec); + std::vector<::resolvo::VersionSetId> version_sets; + for (const std::string& ms : match_specs) + { + alloc_version_set(ms); + } + // Placeholder return value + return ::resolvo::VersionSetId{ 0 }; + } + + // NOTE: This works around some improperly encoded `constrains` in the test data, e.g.: + // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit + // >= 10.2" `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded + // `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" + // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: + // ">=4.5.2" + // Remove any with space after the binary operators + for (const std::string& op : { ">=", "<=", "==", ">", "<", "!=", "=", "==" }) + { + const std::string& bad_op = op + " "; + while (raw_match_spec_str.find(bad_op) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op + + raw_match_spec_str.substr( + raw_match_spec_str.find(bad_op) + bad_op.size() + ); + } + // If start with binary operator, prepend NONE + if (raw_match_spec_str.find(op) == 0) + { + raw_match_spec_str = "NONE " + raw_match_spec_str; + } + } + + const specs::MatchSpec match_spec = specs::MatchSpec::parse(raw_match_spec_str).value(); + // Add the version set to the version set pool + auto id = version_set_pool.alloc(match_spec); + + // Add name to the Name and String pools + const std::string name = match_spec.name().to_string(); + name_pool.alloc(::resolvo::String{ name }); + string_pool.alloc(::resolvo::String{ name }); + + // Add the MatchSpec's string representation to the Name and String pools + const std::string match_spec_str = match_spec.to_string(); + name_pool.alloc(::resolvo::String{ match_spec_str }); + string_pool.alloc(::resolvo::String{ match_spec_str }); + return id; + } + + ::resolvo::SolvableId Database::alloc_solvable(specs::PackageInfo package_info) + { + // Add the solvable to the solvable pool + auto id = solvable_pool.alloc(package_info); + + // Add name to the Name and String pools + const std::string name = package_info.name; + name_pool.alloc(::resolvo::String{ name }); + string_pool.alloc(::resolvo::String{ name }); + + // Add the long string representation of the package to the Name and String pools + const std::string long_str = package_info.long_str(); + name_pool.alloc(::resolvo::String{ long_str }); + string_pool.alloc(::resolvo::String{ long_str }); + + for (auto& dep : package_info.dependencies) + { + alloc_version_set(dep); + } + for (auto& constr : package_info.constrains) + { + alloc_version_set(constr); + } + + // Add the solvable to the name_to_solvable map + const ::resolvo::NameId name_id = name_pool.alloc(::resolvo::String{ package_info.name }); + name_to_solvable[name_id].push_back(id); + + return id; + } + + std::pair + Database::find_highest_version(::resolvo::VersionSetId version_set_id) + { + // If the version set has already been computed, return it. + if (version_set_to_max_version_and_track_features_numbers.find(version_set_id) + != version_set_to_max_version_and_track_features_numbers.end()) + { + return version_set_to_max_version_and_track_features_numbers[version_set_id]; + } + + const specs::MatchSpec match_spec = version_set_pool[version_set_id]; + + const std::string& name = match_spec.name().to_string(); + + auto name_id = name_pool.alloc(::resolvo::String{ name }); + + auto solvables = name_to_solvable[name_id]; + + auto filtered = filter_candidates(solvables, version_set_id, false); + + specs::Version max_version = specs::Version(); + size_t max_version_n_track_features = 0; + + for (auto& solvable_id : filtered) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + const auto version = specs::Version::parse(package_info.version).value(); + if (version == max_version) + { + max_version_n_track_features = std::min( + max_version_n_track_features, + package_info.track_features.size() + ); + } + if (version > max_version) + { + max_version = version; + max_version_n_track_features = package_info.track_features.size(); + } + } + + auto val = std::make_pair(max_version, max_version_n_track_features); + version_set_to_max_version_and_track_features_numbers[version_set_id] = val; + return val; + } + +} // namespace mamba::solver::resolvo diff --git a/libmamba/src/solver/resolvo/solver.cpp b/libmamba/src/solver/resolvo/solver.cpp new file mode 100644 index 0000000000..0c8d889abf --- /dev/null +++ b/libmamba/src/solver/resolvo/solver.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#include "mamba/solver/resolvo/database.hpp" +#include "mamba/solver/resolvo/solver.hpp" +#include "mamba/util/variant_cmp.hpp" + +namespace mamba::solver::resolvo +{ + namespace + { + /** + * An arbitrary comparison function to get determinist output. + */ + auto make_request_cmp() + { + return util::make_variant_cmp( + /** index_cmp= */ + [](auto lhs, auto rhs) { return lhs < rhs; }, + /** alternative_cmp= */ + [](const auto& lhs, const auto& rhs) + { + using Itm = std::decay_t; + if constexpr (!std::is_same_v) + { + return lhs.spec.name().to_string() < rhs.spec.name().to_string(); + } + return false; + } + ); + } + + auto request_to_requirements(const Request& request, Database& database) + -> std::vector<::resolvo::VersionSetId> + { + std::vector<::resolvo::VersionSetId> requirements; + requirements.reserve(request.jobs.size()); + + for (const auto& job : request.jobs) + { + std::visit( + [&](const auto& j) + { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + requirements.push_back( + database.alloc_version_set(j.spec.name().to_string()) + ); + } + }, + job + ); + } + return requirements; + } + + auto request_to_constraints(const Request& request, Database& database) + -> std::vector<::resolvo::VersionSetId> + { + std::vector<::resolvo::VersionSetId> constraints; + constraints.reserve(request.jobs.size()); + + for (const auto& job : request.jobs) + { + std::visit( + [&](const auto& j) + { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + constraints.push_back(database.alloc_version_set(j.spec.name().to_string() + )); + } + }, + job + ); + } + return constraints; + } + + auto + result_to_solution(const ::resolvo::Vector<::resolvo::SolvableId>& result, Database& database, const Request&) + -> Solution + { + Solution solution; + solution.actions.reserve(result.size()); + + for (const auto& solvable_id : result) + { + const auto& solvable = database.solvable_pool[solvable_id]; + specs::PackageInfo pkg; + pkg.name = solvable.name; + pkg.version = solvable.version; + pkg.build_string = solvable.build_string; + pkg.build_number = solvable.build_number; + pkg.channel = solvable.channel; + pkg.md5 = solvable.md5; + pkg.sha256 = solvable.sha256; + pkg.track_features = solvable.track_features; + pkg.dependencies = solvable.dependencies; + pkg.constrains = solvable.constrains; + pkg.timestamp = solvable.timestamp; + pkg.license = solvable.license; + pkg.size = solvable.size; + + solution.actions.emplace_back(Solution::Install{ std::move(pkg) }); + } + + return solution; + } + } + + auto Solver::solve_impl(Database& database, const Request& request) -> expected_t + { + auto requirements = request_to_requirements(request, database); + auto constraints = request_to_constraints(request, database); + ::resolvo::Vector<::resolvo::SolvableId> result; + + ::resolvo::Vector<::resolvo::VersionSetId> req_vec; + for (const auto& req : requirements) + { + req_vec.push_back(req); + } + + ::resolvo::Vector<::resolvo::VersionSetId> constr_vec; + for (const auto& constr : constraints) + { + constr_vec.push_back(constr); + } + + ::resolvo::String reason = ::resolvo::solve(database, req_vec, constr_vec, result); + + if (reason != "") + { + // Get the length from a string view of the reason + std::string_view reason_str_view = reason; + std::string reason_str(reason.data(), reason_str_view.size()); + return make_unexpected(reason_str, mamba_error_code::satisfiablitity_error); + } + + return Outcome{ result_to_solution(result, database, request) }; + } + + auto Solver::solve(Database& database, Request&& request) -> expected_t + { + if (request.flags.order_request) + { + std::sort(request.jobs.begin(), request.jobs.end(), make_request_cmp()); + } + return solve_impl(database, request); + } + + auto Solver::solve(Database& database, const Request& request) -> expected_t + { + if (request.flags.order_request) + { + auto sorted_request = request; + std::sort(sorted_request.jobs.begin(), sorted_request.jobs.end(), make_request_cmp()); + return solve_impl(database, sorted_request); + } + return solve_impl(database, request); + } +} diff --git a/libmambapy/src/libmambapy/bindings/expected_caster.hpp b/libmambapy/src/libmambapy/bindings/expected_caster.hpp index b8c941c48c..af1e329c25 100644 --- a/libmambapy/src/libmambapy/bindings/expected_caster.hpp +++ b/libmambapy/src/libmambapy/bindings/expected_caster.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,128 @@ namespace PYBIND11_NAMESPACE } PYBIND11_TYPE_CASTER(value_type, const_name("None")); + + /** + * A caster for std::variant with reference_wrapper types. + */ + template + struct type_caster...>> + { + using value_type = std::variant...>; + + bool load(handle src, bool convert) + { + return false; // We don't support loading from Python + } + + static handle cast(const value_type& src, return_value_policy policy, handle parent) + { + return std::visit( + [policy, parent](const auto& v) -> handle + { + using T = std::decay_t; + return make_caster::cast(v.get(), policy, parent); + }, + src + ); + } + + PYBIND11_TYPE_CASTER(value_type, _("variant")); + }; + + /** + * A caster for std::reference_wrapper. + */ + template + struct type_caster> + { + using value_type = std::reference_wrapper; + + bool load(handle src, bool convert) + { + return false; // We don't support loading from Python + } + + static handle cast(const value_type& src, return_value_policy policy, handle parent) + { + return make_caster::cast(src.get(), policy, parent); + } + + PYBIND11_TYPE_CASTER(value_type, _("reference_wrapper")); + }; + + /** + * Specialization for the specific variant type used in the database. + */ + template <> + struct type_caster, + std::reference_wrapper>> + { + using value_type = std::variant< + std::reference_wrapper, + std::reference_wrapper>; + + bool load(handle src, bool convert) + { + return false; // We don't support loading from Python + } + + static handle cast(const value_type& src, return_value_policy policy, handle parent) + { + return std::visit( + [policy, parent](const auto& v) -> handle + { + using T = std::decay_t; + return make_caster::cast(v.get(), policy, parent); + }, + src + ); + } + + PYBIND11_TYPE_CASTER(value_type, _("variant")); + }; + + /** + * A caster for mamba::Context. + */ + template <> + struct type_caster + { + using value_type = mamba::Context; + + bool load(handle src, bool convert) + { + return false; // We don't support loading from Python + } + + static handle cast(const value_type& src, return_value_policy policy, handle parent) + { + return make_caster::cast(src, policy, parent); + } + + PYBIND11_TYPE_CASTER(value_type, _("Context")); + }; + + /** + * A caster for mamba::PrefixData. + */ + template <> + struct type_caster + { + using value_type = mamba::PrefixData; + + bool load(handle src, bool convert) + { + return false; // We don't support loading from Python + } + + static handle cast(const value_type& src, return_value_policy policy, handle parent) + { + return make_caster::cast(src, policy, parent); + } + + PYBIND11_TYPE_CASTER(value_type, _("PrefixData")); }; } } diff --git a/libmambapy/src/libmambapy/bindings/solver.cpp b/libmambapy/src/libmambapy/bindings/solver.cpp index 6bc6510c30..79cee82723 100644 --- a/libmambapy/src/libmambapy/bindings/solver.cpp +++ b/libmambapy/src/libmambapy/bindings/solver.cpp @@ -13,6 +13,7 @@ #include "bind_utils.hpp" #include "bindings.hpp" +#include "expected_caster.hpp" #include "flat_set_caster.hpp" namespace mamba::solver diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 0c9cbe3cb2..e674f931e4 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -107,10 +107,13 @@ update_self(Configuration& config, const std::optional& version) mamba::MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); - auto exp_loaded = load_channels(ctx, channel_context, database, package_caches); - if (!exp_loaded) + // Create a variant containing the database + std::variant db_variant = std::move(database + ); + auto exp_load = load_channels(ctx, channel_context, db_variant, package_caches); + if (!exp_load) { - throw exp_loaded.error(); + throw exp_load.error(); } auto matchspec = specs::MatchSpec::parse( From 44b563e76933908fd080cea8b76381700e867742 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 15:10:54 +0200 Subject: [PATCH 33/59] Use `to_string` instead of `str` Signed-off-by: Julien Jerphanion --- libmamba/src/api/update.cpp | 4 +-- .../tests/src/solver/resolvo/test_solver.cpp | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 1068470438..94dd7229f4 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -67,8 +67,8 @@ namespace mamba // We use `spec_names` here because `specs` contain more info than just // the spec name. // Therefore, the search later and comparison (using `specs`) with - // MatchSpec.name().str() in `hist_map` second elements wouldn't be - // relevant + // MatchSpec.name().to_string() in `hist_map` second elements wouldn't + // be relevant std::vector spec_names; spec_names.reserve(specs.size()); std::transform( diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 82874ac3f1..fc3bd9e64a 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -323,12 +323,12 @@ struct PackageDatabase : public DependencyProvider auto id = version_set_pool.alloc(match_spec); // Add name to the Name and String pools - const std::string name = match_spec.name().str(); + const std::string name = match_spec.name().to_string(); name_pool.alloc(String{ name }); string_pool.alloc(String{ name }); // Add the MatchSpec's string representation to the Name and String pools - const std::string match_spec_str = match_spec.str(); + const std::string match_spec_str = match_spec.to_string(); name_pool.alloc(String{ match_spec_str }); string_pool.alloc(String{ match_spec_str }); return id; @@ -434,7 +434,7 @@ struct PackageDatabase : public DependencyProvider String display_version_set(VersionSetId version_set) override { const MatchSpec match_spec = version_set_pool[version_set]; - return String{ match_spec.str() }; + return String{ match_spec.to_string() }; } /** @@ -452,9 +452,9 @@ struct PackageDatabase : public DependencyProvider NameId version_set_name(VersionSetId version_set_id) override { const MatchSpec match_spec = version_set_pool[version_set_id]; - // std::cout << "Getting name id for version_set_id " << match_spec.name().str() << + // std::cout << "Getting name id for version_set_id " << match_spec.name().to_string() << // std::endl; - return name_pool[String{ match_spec.name().str() }]; + return name_pool[String{ match_spec.name().to_string() }]; } /** @@ -491,7 +491,7 @@ struct PackageDatabase : public DependencyProvider const MatchSpec match_spec = version_set_pool[version_set_id]; - const std::string& name = match_spec.name().str(); + const std::string& name = match_spec.name().to_string(); auto name_id = name_pool.alloc(String{ name }); @@ -568,7 +568,7 @@ struct PackageDatabase : public DependencyProvider { // TODO: have a VersionID to NameID mapping instead MatchSpec ms = MatchSpec::parse(dep_a).value(); - const std::string& name = ms.name().str(); + const std::string& name = ms.name().to_string(); auto name_id = name_pool.alloc(String{ name }); a_deps[name_id] = version_set_pool[ms]; @@ -577,7 +577,7 @@ struct PackageDatabase : public DependencyProvider { // TODO: have a VersionID to NameID mapping instead MatchSpec ms = MatchSpec::parse(dep_b).value(); - const std::string& name = ms.name().str(); + const std::string& name = ms.name().to_string(); auto name_id = name_pool.alloc(String{ name }); b_deps[name_id] = version_set_pool[ms]; @@ -627,7 +627,7 @@ struct PackageDatabase : public DependencyProvider MatchSpec match_spec = version_set_pool[version_set_id]; Vector filtered; - // std::cout << "Candidates to filter " << match_spec.str() << std::endl; + // std::cout << "Candidates to filter " << match_spec.to_string() << std::endl; // // for(auto& solvable_id : candidates) { // const PackageInfo& package_info = solvable_pool[solvable_id]; @@ -660,7 +660,7 @@ struct PackageDatabase : public DependencyProvider } } } - // std::cout << "Filtered candidates for " << match_spec.str() << std::endl; + // std::cout << "Filtered candidates for " << match_spec.to_string() << std::endl; // // for(auto& solvable_id : filtered) { // const PackageInfo& package_info = solvable_pool[solvable_id]; @@ -1150,16 +1150,19 @@ TEST_CASE("solver::resolvo") REQUIRE(deps.constrains.size() == 0); REQUIRE( - database.version_set_pool[deps.requirements[0]].str() == "numpy[version=\">=1.20.0,<2.0a0\"]" + database.version_set_pool[deps.requirements[0]].to_string() + == "numpy[version=\">=1.20.0,<2.0a0\"]" ); REQUIRE( - database.version_set_pool[deps.requirements[1]].str() == "scipy[version=\">=1.6.0,<2.0a0\"]" + database.version_set_pool[deps.requirements[1]].to_string() + == "scipy[version=\">=1.6.0,<2.0a0\"]" ); REQUIRE( - database.version_set_pool[deps.requirements[2]].str() == "joblib[version=\">=1.0.1,<2.0a0\"]" + database.version_set_pool[deps.requirements[2]].to_string() + == "joblib[version=\">=1.0.1,<2.0a0\"]" ); REQUIRE( - database.version_set_pool[deps.requirements[3]].str() + database.version_set_pool[deps.requirements[3]].to_string() == "threadpoolctl[version=\">=2.1.0,<3.0a0\"]" ); @@ -1678,7 +1681,7 @@ TEST_CASE("Test consistency with libsolv (environment creation)") auto vid = resolvo_db.alloc_version_set("hypothesis"); auto [version, n_track_features] = resolvo_db.find_highest_version(vid); REQUIRE(n_track_features == 0); - std::cout << "Version: " << version.str() << std::endl; + std::cout << "Version: " << version.to_string() << std::endl; REQUIRE(version > Version::parse("6.105.1").value()); } From 8afe9a41ece5dc528ae45e6a771947786980fc2a Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 16:40:26 +0200 Subject: [PATCH 34/59] Define a type for the variant Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/channel_loader.hpp | 3 +- libmamba/include/mamba/core/context.hpp | 1 + .../include/mamba/solver/solver_factory.hpp | 6 ++ libmamba/src/api/channel_loader.cpp | 5 +- libmamba/src/api/configuration.cpp | 8 ++ libmamba/src/api/install.cpp | 26 +++--- libmamba/src/api/repoquery.cpp | 2 +- libmamba/src/api/update.cpp | 3 +- micromamba/src/update.cpp | 93 +++++++++++++------ 9 files changed, 100 insertions(+), 47 deletions(-) diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index fa9cbe2636..45616e7420 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -9,6 +9,7 @@ #include "mamba/core/error_handling.hpp" #include "mamba/solver/resolvo/database.hpp" +#include "mamba/solver/solver_factory.hpp" namespace mamba { @@ -36,7 +37,7 @@ namespace mamba auto load_channels( Context& ctx, ChannelContext& channel_context, - std::variant& database, + solver::DatabaseVariant& database, MultiPackageCache& package_caches ) -> expected_t; diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index 0b13d88851..e5acccaa95 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -105,6 +105,7 @@ namespace mamba bool experimental = false; bool experimental_repodata_parsing = true; bool experimental_matchspec_parsing = false; + bool experimental_resolvo_solver = false; bool debug = false; bool use_uv = false; diff --git a/libmamba/include/mamba/solver/solver_factory.hpp b/libmamba/include/mamba/solver/solver_factory.hpp index 37a4da4dd5..8c9c3376fa 100644 --- a/libmamba/include/mamba/solver/solver_factory.hpp +++ b/libmamba/include/mamba/solver/solver_factory.hpp @@ -8,6 +8,7 @@ #define MAMBA_SOLVER_SOLVER_FACTORY_HPP #include +#include #include "mamba/core/context.hpp" #include "mamba/solver/libsolv/database.hpp" @@ -17,6 +18,11 @@ namespace mamba::solver { + /** + * Type alias for the database variant that can hold either libsolv or resolvo database. + */ + using DatabaseVariant = std::variant; + /** * Create a solver based on the configuration. * diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 2abb7c79a4..0fbf3c7675 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -14,6 +14,7 @@ #include "mamba/core/virtual_packages.hpp" #include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/libsolv/repo_info.hpp" +#include "mamba/solver/solver_factory.hpp" #include "mamba/specs/package_info.hpp" namespace mamba @@ -149,7 +150,7 @@ namespace mamba auto load_channels_impl( Context& ctx, ChannelContext& channel_context, - std::variant& database, + solver::DatabaseVariant& database, MultiPackageCache& package_caches, bool is_retry ) -> expected_t @@ -364,7 +365,7 @@ namespace mamba auto load_channels( Context& ctx, ChannelContext& channel_context, - std::variant& database, + solver::DatabaseVariant& database, MultiPackageCache& package_caches ) -> expected_t { diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index 611e16fe26..27d094d795 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -1410,6 +1410,14 @@ namespace mamba ) .set_env_var_names()); + insert(Configurable("experimental_resolvo_solver", &m_context.experimental_resolvo_solver) + .group("Basic") + .description( // + "Enable the experimental resolvo solver instead of libsolv.\n" + "This is not meant for production" + ) + .set_env_var_names()); + insert(Configurable("debug", &m_context.debug) .group("Basic") .set_env_var_names() diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index b654acc4ad..751c4e5150 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -567,7 +567,7 @@ namespace mamba }; add_spdlog_logger_to_database(libsolv_db); - std::variant db_variant( + solver::DatabaseVariant db_variant( std::in_place_type, std::move(libsolv_db) ); @@ -745,19 +745,17 @@ namespace mamba ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv }; - std::variant - db = ctx.use_resolvo_solver - ? std::variant< - solver::libsolv::Database, - solver::resolvo::Database>(std::in_place_type) - : std::variant( - std::in_place_type, - channel_context.params(), - solver::libsolv::Database::Settings{ - ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } - ); + solver::DatabaseVariant db = ctx.use_resolvo_solver + ? solver::DatabaseVariant(std::in_place_type< + solver::resolvo::Database>) + : solver::DatabaseVariant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); if (!ctx.use_resolvo_solver) { add_spdlog_logger_to_database(std::get(db)); diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index 0096e5ddfa..92b741e0ea 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -36,7 +36,7 @@ namespace mamba config.load(); auto channel_context = ChannelContext::make_conda_compatible(ctx); - static std::variant db( + solver::DatabaseVariant db( std::in_place_type, channel_context.params(), solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 94dd7229f4..d82b013273 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -17,6 +17,7 @@ #include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/libsolv/solver.hpp" #include "mamba/solver/request.hpp" +#include "mamba/solver/solver_factory.hpp" #include "utils.hpp" @@ -156,7 +157,7 @@ namespace mamba populate_context_channels_from_specs(raw_update_specs, ctx); - std::variant db( + solver::DatabaseVariant db( std::in_place_type, channel_context.params(), solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index e674f931e4..861780736f 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -16,6 +16,7 @@ #include "mamba/core/package_database_loader.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util_os.hpp" +#include "mamba/solver/solver_factory.hpp" #include "mamba/util/build.hpp" #ifdef __APPLE__ @@ -56,36 +57,52 @@ set_update_command(CLI::App* subcom, Configuration& config) #ifdef BUILDING_MICROMAMBA namespace { - auto database_has_package(solver::libsolv::Database& database, specs::MatchSpec spec) -> bool + auto database_has_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> bool { bool found = false; - database.for_each_package_matching( - spec, - [&](const auto&) - { - found = true; - return util::LoopControl::Break; - } - ); + if (auto* libsolv_db = std::get_if(&database)) + { + libsolv_db->for_each_package_matching( + spec, + [&](const auto&) + { + found = true; + return util::LoopControl::Break; + } + ); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + // TODO: Implement for resolvo database + throw std::runtime_error("resolvo database not yet supported for self-update"); + } return found; }; - auto database_latest_package(solver::libsolv::Database& database, specs::MatchSpec spec) + auto database_latest_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> std::optional { auto out = std::optional(); - database.for_each_package_matching( - spec, - [&](auto pkg) - { - if (!out - || (specs::Version::parse(pkg.version).value_or(specs::Version()) - > specs::Version::parse(out->version).value_or(specs::Version()))) + if (auto* libsolv_db = std::get_if(&database)) + { + libsolv_db->for_each_package_matching( + spec, + [&](auto pkg) { - out = std::move(pkg); + if (!out + || (specs::Version::parse(pkg.version).value_or(specs::Version()) + > specs::Version::parse(out->version).value_or(specs::Version()))) + { + out = std::move(pkg); + } } - } - ); + ); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + // TODO: Implement for resolvo database + throw std::runtime_error("resolvo database not yet supported for self-update"); + } return out; }; } @@ -102,14 +119,22 @@ update_self(Configuration& config, const std::optional& version) auto channel_context = ChannelContext::make_conda_compatible(ctx); - solver::libsolv::Database database{ channel_context.params() }; - add_spdlog_logger_to_database(database); + auto db_variant = [&]() -> solver::DatabaseVariant + { + if (ctx.experimental_resolvo_solver) + { + return solver::resolvo::Database{}; + } + else + { + solver::libsolv::Database database{ channel_context.params() }; + add_spdlog_logger_to_database(database); + return std::move(database); + } + }(); mamba::MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); - // Create a variant containing the database - std::variant db_variant = std::move(database - ); auto exp_load = load_channels(ctx, channel_context, db_variant, package_caches); if (!exp_load) { @@ -123,11 +148,11 @@ update_self(Configuration& config, const std::optional& version) .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); - auto latest_micromamba = database_latest_package(database, matchspec); + auto latest_micromamba = database_latest_package(db_variant, matchspec); if (!latest_micromamba.has_value()) { - if (database_has_package(database, specs::MatchSpec::parse("micromamba").value())) + if (database_has_package(db_variant, specs::MatchSpec::parse("micromamba").value())) { Console::instance().print( fmt::format("\nYour micromamba version ({}) is already up to date.", umamba::version()) @@ -155,7 +180,19 @@ update_self(Configuration& config, const std::optional& version) ); ctx.download_only = true; - MTransaction t(ctx, database, { latest_micromamba.value() }, package_caches); + auto t = [&]() -> MTransaction + { + if (auto* libsolv_db = std::get_if(&db_variant)) + { + return MTransaction(ctx, *libsolv_db, { latest_micromamba.value() }, package_caches); + } + else if (auto* resolvo_db = std::get_if(&db_variant)) + { + // TODO: Implement for resolvo database + throw std::runtime_error("resolvo database not yet supported for self-update"); + } + throw std::runtime_error("Invalid database variant type"); + }(); auto exp_prefix_data = PrefixData::create(ctx.prefix_params.root_prefix, channel_context); if (!exp_prefix_data) { From 92df45502bc1b82e2330ef1a460be49c869a3a18 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 17:12:02 +0200 Subject: [PATCH 35/59] Use DatabaseVariant for MTransaction and friends Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/core/transaction.hpp | 12 +- libmamba/src/api/install.cpp | 40 ++--- libmamba/src/api/remove.cpp | 11 +- libmamba/src/api/update.cpp | 2 +- libmamba/src/core/transaction.cpp | 170 +++++++++--------- libmamba/tests/src/core/test_env_lockfile.cpp | 3 +- micromamba/src/update.cpp | 2 +- 7 files changed, 115 insertions(+), 125 deletions(-) diff --git a/libmamba/include/mamba/core/transaction.hpp b/libmamba/include/mamba/core/transaction.hpp index b56d7d85e4..9437eb5bcd 100644 --- a/libmamba/include/mamba/core/transaction.hpp +++ b/libmamba/include/mamba/core/transaction.hpp @@ -15,8 +15,8 @@ #include "mamba/core/package_cache.hpp" #include "mamba/core/prefix_data.hpp" #include "mamba/fs/filesystem.hpp" -#include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/solution.hpp" +#include "mamba/solver/solver_factory.hpp" #include "mamba/specs/match_spec.hpp" #include "mamba/specs/package_info.hpp" @@ -36,7 +36,7 @@ namespace mamba MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, std::vector pkgs_to_remove, std::vector pkgs_to_install, MultiPackageCache& caches @@ -44,7 +44,7 @@ namespace mamba MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const solver::Request& request, solver::Solution solution, MultiPackageCache& caches @@ -53,7 +53,7 @@ namespace mamba // Only use if the packages have been solved previously already. MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, std::vector packages, MultiPackageCache& caches ); @@ -90,7 +90,7 @@ namespace mamba MTransaction create_explicit_transaction_from_urls( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const std::vector& urls, MultiPackageCache& package_caches, std::vector& other_specs @@ -98,7 +98,7 @@ namespace mamba MTransaction create_explicit_transaction_from_lockfile( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const fs::u8path& env_lockfile_path, const std::vector& categories, MultiPackageCache& package_caches, diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 751c4e5150..c7bd80207f 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -558,19 +558,15 @@ namespace mamba ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv }; - solver::libsolv::Database libsolv_db{ + solver::libsolv::Database libsolv_db( channel_context.params(), - { - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - }, - }; - add_spdlog_logger_to_database(libsolv_db); - - solver::DatabaseVariant db_variant( - std::in_place_type, - std::move(libsolv_db) + { ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } ); + solver::DatabaseVariant db_variant = std::move(libsolv_db); + + add_spdlog_logger_to_database(std::get(db_variant)); + auto maybe_load = load_channels(ctx, channel_context, db_variant, package_caches); if (!maybe_load) { @@ -647,13 +643,7 @@ namespace mamba auto& solution = std::get(result); Console::instance().json_write({ { "success", true } }); - auto transaction = MTransaction( - ctx, - std::get(db_variant), - request, - solution, - package_caches - ); + auto transaction = MTransaction(ctx, db_variant, request, solution, package_caches); std::vector locks; @@ -729,8 +719,8 @@ namespace mamba namespace { - - // TransactionFunc: (Database& database, MultiPackageCache& package_caches) -> MTransaction + // TransactionFunc: (DatabaseVariant& database, MultiPackageCache& package_caches, ...) -> + // MTransaction template void install_explicit_with_transaction( Context& ctx, @@ -790,11 +780,7 @@ namespace mamba std::vector others; // Note that the Transaction will gather the Solvables, // so they must have been ready in the database's pool before this line - auto transaction = create_transaction( - std::get(db), - pkg_caches, - others - ); + auto transaction = create_transaction(db, pkg_caches, others); std::vector lock_pkgs; @@ -853,7 +839,7 @@ namespace mamba ctx, channel_context, specs, - [&](auto& db, auto& pkg_caches, auto& others) + [&](solver::DatabaseVariant& db, auto& pkg_caches, auto& others) { return create_explicit_transaction_from_urls(ctx, db, specs, pkg_caches, others); }, create_env, remove_prefix_on_failure @@ -885,7 +871,7 @@ namespace mamba ctx, channel_context, {}, - [&](auto& db, auto& pkg_caches, auto& others) + [&](solver::DatabaseVariant& db, auto& pkg_caches, auto& others) { return create_explicit_transaction_from_lockfile( ctx, diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index 5697984a84..d753a2e682 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -188,7 +188,8 @@ namespace mamba pkgs_to_remove.push_back(iter->second); } } - auto transaction = MTransaction(ctx, database, pkgs_to_remove, {}, package_caches); + solver::DatabaseVariant db_variant = std::move(database); + auto transaction = MTransaction(ctx, db_variant, pkgs_to_remove, {}, package_caches); return execute_transaction(transaction); } else @@ -227,15 +228,15 @@ namespace mamba } Console::instance().json_write({ { "success", true } }); - auto transaction = MTransaction( + solver::DatabaseVariant db_variant2 = std::move(database); + auto transaction2 = MTransaction( ctx, - database, + db_variant2, request, std::get(outcome), package_caches ); - - return execute_transaction(transaction); + return execute_transaction(transaction2); } } } diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index d82b013273..9b8eabeff9 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -238,7 +238,7 @@ namespace mamba Console::instance().json_write({ { "success", true } }); auto transaction = MTransaction( ctx, - std::get(db), + db, request, std::get(outcome), package_caches diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 111314a8b4..06bd8dda23 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -51,21 +51,80 @@ namespace mamba && caches.get_tarball_path(pkg_info).empty(); } - // TODO duplicated function, consider moving it to Pool - auto database_has_package(solver::libsolv::Database& database, const specs::MatchSpec& spec) + auto database_has_package(solver::DatabaseVariant& database, const specs::MatchSpec& spec) -> bool { bool found = false; - database.for_each_package_matching( - spec, - [&](const auto&) + if (auto* libsolv_db = std::get_if(&database)) + { + libsolv_db->for_each_package_matching( + spec, + [&](const auto&) + { + found = true; + return util::LoopControl::Break; + } + ); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + // TODO: Implement for resolvo database + throw std::runtime_error("Package matching not yet implemented for resolvo database"); + } + return found; + } + + auto installed_python(const solver::DatabaseVariant& database) -> std::optional + { + if (auto* libsolv_db = std::get_if(&database)) + { + auto out = std::optional(); + if (auto repo = libsolv_db->installed_repo()) + { + libsolv_db->for_each_package_in_repo( + *repo, + [&](specs::PackageInfo&& pkg) + { + if (pkg.name == "python") + { + out = pkg.version; + return util::LoopControl::Break; + } + return util::LoopControl::Continue; + } + ); + } + return out; + } + else if (auto* resolvo_db = std::get_if(&database)) + { + // TODO: Implement for resolvo database + throw std::runtime_error("Python version lookup not yet implemented for resolvo database" + ); + } + return std::nullopt; + } + + auto + find_python_version(const solver::Solution& solution, const solver::DatabaseVariant& database) + -> std::pair + { + auto old_python = installed_python(database); + auto new_python = std::optional(); + + for_each_to_install( + solution.actions, + [&](const auto& pkg) { - found = true; - return util::LoopControl::Break; + if (pkg.name == "python") + { + new_python = pkg.version; + } } ); - return found; - }; + + return { new_python.value_or(""), old_python.value_or("") }; + } auto explicit_spec(const specs::PackageInfo& pkg) -> specs::MatchSpec { @@ -86,55 +145,6 @@ namespace mamba } return out; } - - auto installed_python(const solver::libsolv::Database& database) - -> std::optional - { - // TODO combine Repo and MatchSpec search API in Pool - auto out = std::optional(); - if (auto repo = database.installed_repo()) - { - database.for_each_package_in_repo( - *repo, - [&](specs::PackageInfo&& pkg) - { - if (pkg.name == "python") - { - out = std::move(pkg); - return util::LoopControl::Break; - } - return util::LoopControl::Continue; - } - ); - } - return out; - } - - auto - find_python_version(const solver::Solution& solution, const solver::libsolv::Database& database) - -> std::pair - { - // We need to find the python version that will be there after this - // Transaction is finished in order to compile the noarch packages correctly, - - // We need to look into installed packages in case we are not installing a new python - // version but keeping the current one. - // Could also be written in term of PrefixData. - std::string installed_py_ver = {}; - if (auto pkg = installed_python(database)) - { - installed_py_ver = pkg->version; - LOG_INFO << "Found python in installed packages " << installed_py_ver; - } - - std::string new_py_ver = installed_py_ver; - if (auto py = solver::find_new_python_in_solution(solution)) - { - new_py_ver = py->get().version; - } - - return { std::move(new_py_ver), std::move(installed_py_ver) }; - } } MTransaction::MTransaction(const CommandParams& command_params, MultiPackageCache& caches) @@ -145,7 +155,7 @@ namespace mamba MTransaction::MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, std::vector pkgs_to_remove, std::vector pkgs_to_install, MultiPackageCache& caches @@ -191,26 +201,13 @@ namespace mamba } m_solution.actions.reserve(pkgs_to_install.size() + pkgs_to_remove.size()); - - std::transform( - std::move_iterator(pkgs_to_install.begin()), - std::move_iterator(pkgs_to_install.end()), - std::back_insert_iterator(m_solution.actions), - [](specs::PackageInfo&& pkg) { return solver::Solution::Install{ std::move(pkg) }; } - ); - - std::transform( - std::move_iterator(pkgs_to_remove.begin()), - std::move_iterator(pkgs_to_remove.end()), - std::back_insert_iterator(m_solution.actions), - [](specs::PackageInfo&& pkg) { return solver::Solution::Remove{ std::move(pkg) }; } - ); - - // if no action required, don't even start logging them - if (!empty()) + for (auto& pkg : pkgs_to_install) { - Console::instance().json_down("actions"); - Console::instance().json_write({ { "PREFIX", ctx.prefix_params.target_prefix.string() } }); + m_solution.actions.push_back(solver::Solution::Install{ std::move(pkg) }); + } + for (auto& pkg : pkgs_to_remove) + { + m_solution.actions.push_back(solver::Solution::Remove{ std::move(pkg) }); } m_py_versions = find_python_version(m_solution, database); @@ -218,7 +215,7 @@ namespace mamba MTransaction::MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const solver::Request& request, solver::Solution solution, MultiPackageCache& caches @@ -272,7 +269,7 @@ namespace mamba MTransaction::MTransaction( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, std::vector packages, MultiPackageCache& caches ) @@ -1076,8 +1073,13 @@ namespace mamba t.print(out); } - MTransaction - create_explicit_transaction_from_urls(const Context& ctx, solver::libsolv::Database& database, const std::vector& urls, MultiPackageCache& package_caches, std::vector&) + MTransaction create_explicit_transaction_from_urls( + const Context& ctx, + solver::DatabaseVariant& database, + const std::vector& urls, + MultiPackageCache& package_caches, + std::vector& other_specs + ) { std::vector specs_to_install = {}; specs_to_install.reserve(urls.size()); @@ -1097,7 +1099,7 @@ namespace mamba MTransaction create_explicit_transaction_from_lockfile( const Context& ctx, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const fs::u8path& env_lockfile_path, const std::vector& categories, MultiPackageCache& package_caches, @@ -1161,7 +1163,7 @@ namespace mamba ); } - return MTransaction{ ctx, database, std::move(conda_packages), package_caches }; + return MTransaction(ctx, database, conda_packages, package_caches); } } // namespace mamba diff --git a/libmamba/tests/src/core/test_env_lockfile.cpp b/libmamba/tests/src/core/test_env_lockfile.cpp index b7ab316757..1dd3660c39 100644 --- a/libmamba/tests/src/core/test_env_lockfile.cpp +++ b/libmamba/tests/src/core/test_env_lockfile.cpp @@ -154,9 +154,10 @@ namespace mamba [&](std::vector categories, size_t num_conda, size_t num_pip) { std::vector other_specs; + solver::DatabaseVariant db_variant = std::move(db); auto transaction = create_explicit_transaction_from_lockfile( ctx, - db, + db_variant, lockfile_path, categories, pkg_cache, diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 861780736f..3679ef97cb 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -184,7 +184,7 @@ update_self(Configuration& config, const std::optional& version) { if (auto* libsolv_db = std::get_if(&db_variant)) { - return MTransaction(ctx, *libsolv_db, { latest_micromamba.value() }, package_caches); + return MTransaction(ctx, db_variant, { latest_micromamba.value() }, package_caches); } else if (auto* resolvo_db = std::get_if(&db_variant)) { From 2590c52973056a74eec6104621f82f42455ffb40 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 17:27:33 +0200 Subject: [PATCH 36/59] Deduplicate configurable for using resolvo Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/core/context.hpp | 2 -- libmamba/include/mamba/solver/solver_factory.hpp | 2 +- libmamba/src/api/configuration.cpp | 10 +--------- libmamba/src/api/install.cpp | 4 ++-- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index e5acccaa95..4c9b84e191 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -265,8 +265,6 @@ namespace mamba bool repodata_use_zst = true; std::vector repodata_has_zst = { "https://conda.anaconda.org/conda-forge" }; - bool use_resolvo_solver = false; - // FIXME: Should not be stored here // Notice that we cannot build this map directly from mirrored_channels, // since we need to add a single "mirror" for non mirrored channels diff --git a/libmamba/include/mamba/solver/solver_factory.hpp b/libmamba/include/mamba/solver/solver_factory.hpp index 8c9c3376fa..b1c14c0c1b 100644 --- a/libmamba/include/mamba/solver/solver_factory.hpp +++ b/libmamba/include/mamba/solver/solver_factory.hpp @@ -32,7 +32,7 @@ namespace mamba::solver template auto create_solver(const Context& ctx) { - if (ctx.use_resolvo_solver) + if (ctx.experimental_resolvo_solver) { return std::make_unique(); } diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index 27d094d795..74355d8e99 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -1410,14 +1410,6 @@ namespace mamba ) .set_env_var_names()); - insert(Configurable("experimental_resolvo_solver", &m_context.experimental_resolvo_solver) - .group("Basic") - .description( // - "Enable the experimental resolvo solver instead of libsolv.\n" - "This is not meant for production" - ) - .set_env_var_names()); - insert(Configurable("debug", &m_context.debug) .group("Basic") .set_env_var_names() @@ -1626,7 +1618,7 @@ namespace mamba When enabled, use the experimental resolvo solver instead of libsolv. This is an experimental feature and may not be fully functional.)")) .set_post_merge_hook([&](bool& value) - { m_context.use_resolvo_solver = value; })); + { m_context.experimental_resolvo_solver = value; })); insert(Configurable("explicit_install", false) .group("Solver") diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index c7bd80207f..ad0644f571 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -735,7 +735,7 @@ namespace mamba ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv }; - solver::DatabaseVariant db = ctx.use_resolvo_solver + solver::DatabaseVariant db = ctx.experimental_resolvo_solver ? solver::DatabaseVariant(std::in_place_type< solver::resolvo::Database>) : solver::DatabaseVariant( @@ -746,7 +746,7 @@ namespace mamba ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv } ); - if (!ctx.use_resolvo_solver) + if (!ctx.experimental_resolvo_solver) { add_spdlog_logger_to_database(std::get(db)); } From 8350a725a55e7ab617ed51c50678e0000e9491b1 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 17:41:57 +0200 Subject: [PATCH 37/59] Use DatabaseVariant for repoquery Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/repoquery.hpp | 7 +- libmamba/src/api/repoquery.cpp | 125 +++++++++++------------ 2 files changed, 65 insertions(+), 67 deletions(-) diff --git a/libmamba/include/mamba/api/repoquery.hpp b/libmamba/include/mamba/api/repoquery.hpp index 4d55661388..e6dd0a2ec8 100644 --- a/libmamba/include/mamba/api/repoquery.hpp +++ b/libmamba/include/mamba/api/repoquery.hpp @@ -10,6 +10,7 @@ #include "mamba/api/configuration.hpp" #include "mamba/core/query.hpp" +#include "mamba/solver/solver_factory.hpp" namespace mamba { @@ -23,7 +24,7 @@ namespace mamba }; [[nodiscard]] auto make_repoquery( - solver::libsolv::Database& database, + solver::DatabaseVariant& database, QueryType type, QueryResultFormat format, const std::vector& queries, @@ -32,6 +33,10 @@ namespace mamba std::ostream& out ) -> bool; + [[nodiscard]] auto + repoquery_init(Context& ctx, Configuration& config, QueryResultFormat format, bool use_local) + -> solver::DatabaseVariant; + [[nodiscard]] auto repoquery( Configuration& config, QueryType type, diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index 92b741e0ea..41567f07e1 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -20,81 +20,69 @@ namespace mamba { - namespace + auto repoquery_init(Context& ctx, Configuration& config, QueryResultFormat format, bool use_local) + -> solver::DatabaseVariant { - auto - repoquery_init(Context& ctx, Configuration& config, QueryResultFormat format, bool use_local) - -> solver::libsolv::Database& - { - config.at("use_target_prefix_fallback").set_value(true); - config.at("use_default_prefix_fallback").set_value(true); - config.at("use_root_prefix_fallback").set_value(true); - config.at("target_prefix_checks") - .set_value( - MAMBA_ALLOW_EXISTING_PREFIX | MAMBA_ALLOW_MISSING_PREFIX | MAMBA_ALLOW_NOT_ENV_PREFIX - ); - config.load(); - - auto channel_context = ChannelContext::make_conda_compatible(ctx); - solver::DatabaseVariant db( - std::in_place_type, - channel_context.params(), - solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } + config.at("use_target_prefix_fallback").set_value(true); + config.at("use_default_prefix_fallback").set_value(true); + config.at("use_root_prefix_fallback").set_value(true); + config.at("target_prefix_checks") + .set_value( + MAMBA_ALLOW_EXISTING_PREFIX | MAMBA_ALLOW_MISSING_PREFIX | MAMBA_ALLOW_NOT_ENV_PREFIX ); - add_spdlog_logger_to_database(std::get(db)); + config.load(); + + auto channel_context = ChannelContext::make_conda_compatible(ctx); + solver::DatabaseVariant db( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); + add_spdlog_logger_to_database(std::get(db)); - // bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds); - MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); - if (use_local) + // bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds); + MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); + if (use_local) + { + if (format != QueryResultFormat::Json) + { + Console::stream() << "Using local repodata..." << std::endl; + } + auto exp_prefix_data = PrefixData::create(ctx.prefix_params.target_prefix, channel_context); + if (!exp_prefix_data) { - if (format != QueryResultFormat::Json) - { - Console::stream() << "Using local repodata..." << std::endl; - } - auto exp_prefix_data = PrefixData::create( - ctx.prefix_params.target_prefix, - channel_context - ); - if (!exp_prefix_data) - { - // TODO: propagate tl::expected mechanism - throw std::runtime_error(exp_prefix_data.error().what()); - } - PrefixData& prefix_data = exp_prefix_data.value(); + // TODO: propagate tl::expected mechanism + throw std::runtime_error(exp_prefix_data.error().what()); + } + PrefixData& prefix_data = exp_prefix_data.value(); - load_installed_packages_in_database( - ctx, - std::get(db), - prefix_data - ); + load_installed_packages_in_database(ctx, std::get(db), prefix_data); - if (format != QueryResultFormat::Json) - { - Console::stream() - << "Loaded current active prefix: " << ctx.prefix_params.target_prefix - << std::endl; - } + if (format != QueryResultFormat::Json) + { + Console::stream() << "Loaded current active prefix: " + << ctx.prefix_params.target_prefix << std::endl; } - else + } + else + { + if (format != QueryResultFormat::Json) { - if (format != QueryResultFormat::Json) - { - Console::stream() << "Getting repodata from channels..." << std::endl; - } - auto exp_load = load_channels(ctx, channel_context, db, package_caches); - if (!exp_load) - { - throw std::runtime_error(exp_load.error().what()); - } + Console::stream() << "Getting repodata from channels..." << std::endl; + } + auto exp_load = load_channels(ctx, channel_context, db, package_caches); + if (!exp_load) + { + throw std::runtime_error(exp_load.error().what()); } - return std::get(db); } + return db; } auto make_repoquery( - solver::libsolv::Database& database, + solver::DatabaseVariant& database, QueryType type, QueryResultFormat format, const std::vector& queries, @@ -103,9 +91,14 @@ namespace mamba std::ostream& out ) -> bool { + if (std::holds_alternative(database)) + { + throw std::runtime_error("repoquery does not support the resolvo solver yet"); + } + auto& libsolv_db = std::get(database); if (type == QueryType::Search) { - auto res = Query::find(database, queries); + auto res = Query::find(libsolv_db, queries); switch (format) { case QueryResultFormat::Json: @@ -126,7 +119,7 @@ namespace mamba throw std::invalid_argument("Only one query supported for 'depends'."); } auto res = Query::depends( - database, + libsolv_db, queries.front(), /* tree= */ format == QueryResultFormat::Tree || format == QueryResultFormat::RecursiveTable @@ -153,7 +146,7 @@ namespace mamba throw std::invalid_argument("Only one query supported for 'whoneeds'."); } auto res = Query::whoneeds( - database, + libsolv_db, queries.front(), /* tree= */ format == QueryResultFormat::Tree || format == QueryResultFormat::RecursiveTable @@ -195,7 +188,7 @@ namespace mamba ) { auto& ctx = config.context(); - auto& db = repoquery_init(ctx, config, format, use_local); + auto db = repoquery_init(ctx, config, format, use_local); return make_repoquery( db, type, From 05eb4649d4220d2639c22ac22a73d42f808692de Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 17:48:52 +0200 Subject: [PATCH 38/59] Log solver used Signed-off-by: Julien Jerphanion --- libmamba/src/core/transaction.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 06bd8dda23..0fb33591be 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -804,6 +804,8 @@ namespace mamba Console::instance().print("Transaction\n"); Console::stream() << " Prefix: " << ctx.prefix_params.target_prefix.string() << "\n"; + Console::stream() << " Solver: " + << (ctx.experimental_resolvo_solver ? "resolvo" : "libsolv") << "\n"; // check size of transaction if (empty()) From 0926f055f6cb40624d0bfc569b390387408fda78 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Mon, 19 May 2025 18:07:21 +0200 Subject: [PATCH 39/59] WIP solver integration Signed-off-by: Julien Jerphanion --- libmamba/src/api/install.cpp | 61 +++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index ad0644f571..f869c4a443 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -607,38 +607,55 @@ namespace mamba // Console stream prints on destruction } - auto outcome = solver::libsolv::Solver().solve( - std::get(db_variant), - request, - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv - ); + using LibsolvOutcome = std::variant; + auto outcome = ctx.experimental_resolvo_solver + ? solver::resolvo::Solver() + .solve(std::get(db_variant), request) + .map( + [](auto&& result) -> LibsolvOutcome + { + // resolvo only returns Solution + return std::get(result); + } + ) + : solver::libsolv::Solver().solve( + std::get(db_variant), + request, + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv + ); if (!outcome.has_value()) { throw std::runtime_error(outcome.error().what()); } auto& result = outcome.value(); - if (auto* unsolvable = std::get_if(&result)) + + // If resolvo is used, we don't need to handle UnSolvable + if (!ctx.experimental_resolvo_solver) { - unsolvable->explain_problems_to( - std::get(db_variant), - std::cout, - mamba::solver::ProblemsMessageFormat{} - ); - if (ctx.output_params.json) + if (auto* unsolvable = std::get_if(&result)) { - nlohmann::json j; - j["success"] = false; - j["solver_problems"] = unsolvable->problems( - std::get(db_variant) + unsolvable->explain_problems_to( + std::get(db_variant), + std::cout, + mamba::solver::ProblemsMessageFormat{} + ); + if (ctx.output_params.json) + { + nlohmann::json j; + j["success"] = false; + j["solver_problems"] = unsolvable->problems( + std::get(db_variant) + ); + Console::instance().json_write(j); + } + throw mamba_error( + "Could not solve for environment specs", + mamba_error_code::satisfiablitity_error ); - Console::instance().json_write(j); } - throw mamba_error( - "Could not solve for environment specs", - mamba_error_code::satisfiablitity_error - ); } auto& solution = std::get(result); From d727e7958d2e61a400e5b3aa6e3e0f3a2992a7bc Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 12:12:43 +0200 Subject: [PATCH 40/59] Implement for resolvo Signed-off-by: Julien Jerphanion --- libmamba/src/api/channel_loader.cpp | 83 +++++++++++++++++++++++++---- libmamba/src/api/repoquery.cpp | 12 ++++- libmamba/src/api/update.cpp | 12 ++++- micromamba/src/update.cpp | 40 +++++++------- 4 files changed, 118 insertions(+), 29 deletions(-) diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 0fbf3c7675..dd1e1d034d 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -16,6 +16,7 @@ #include "mamba/solver/libsolv/repo_info.hpp" #include "mamba/solver/solver_factory.hpp" #include "mamba/specs/package_info.hpp" +#include "mamba/util/string.hpp" namespace mamba { @@ -227,9 +228,7 @@ namespace mamba } else if (auto* resolvo_db = std::get_if(&database)) { - (void) resolvo_db; // Silence unused variable warning - // TODO: Implement this for resolvo - throw std::runtime_error("Offline mode not supported with resolvo solver yet"); + resolvo_db->add_repo_from_packages(packages, "packages"); } } @@ -283,9 +282,25 @@ namespace mamba } else if (auto* resolvo_db = std::get_if(&database)) { - (void) resolvo_db; // Silence unused variable warning - // TODO: Implement this for resolvo - throw std::runtime_error("Offline mode not supported with resolvo solver yet"); + // For resolvo, we need to create a vector of PackageInfo objects from the + // pkgs_dir + std::vector offline_packages; + for (const auto& entry : fs::directory_iterator(c)) + { + if (entry.path().extension() == ".tar.bz2" + || entry.path().extension() == ".conda") + { + auto pkg_info = specs::PackageInfo::from_url(entry.path().string()) + .or_else([](specs::ParseError&& err) + { throw std::move(err); }) + .value(); + offline_packages.push_back(std::move(pkg_info)); + } + } + if (!offline_packages.empty()) + { + resolvo_db->add_repo_from_packages(offline_packages, "offline_packages"); + } } } } @@ -337,9 +352,59 @@ namespace mamba } else if (auto* resolvo_db = std::get_if(&database)) { - (void) resolvo_db; // Silence unused variable warning - // TODO: Implement this for resolvo - throw std::runtime_error("Loading subdirs not supported with resolvo solver yet"); + // For resolvo, we need to load the repodata.json file and add it to the + // database + auto repodata_json = subdir.valid_json_cache_path(); + if (!repodata_json) + { + if (is_retry) + { + std::stringstream ss; + ss << "Could not load repodata.json for " << subdir.name() + << " after retry." + << "Please check repodata source. Exiting." << std::endl; + error_list.push_back( + mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) + ); + } + else + { + LOG_WARNING << "Could not load repodata.json for " << subdir.name() + << ". Deleting cache, and retrying."; + subdir.clear_valid_cache_files(); + loading_failed = true; + } + continue; + } + + try + { + resolvo_db->add_repo_from_repodata_json( + repodata_json.value(), + util::rsplit(subdir.metadata().url(), "/", 1).front(), + subdir.channel_id() + ); + } + catch (const std::exception& e) + { + if (is_retry) + { + std::stringstream ss; + ss << "Could not load repodata.json for " << subdir.name() + << " after retry: " << e.what() + << ". Please check repodata source. Exiting." << std::endl; + error_list.push_back( + mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) + ); + } + else + { + LOG_WARNING << "Could not load repodata.json for " << subdir.name() + << ": " << e.what() << ". Deleting cache, and retrying."; + subdir.clear_valid_cache_files(); + loading_failed = true; + } + } } } diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index 41567f07e1..00da72ff79 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -58,7 +58,17 @@ namespace mamba } PrefixData& prefix_data = exp_prefix_data.value(); - load_installed_packages_in_database(ctx, std::get(db), prefix_data); + load_installed_packages_in_database( + ctx, + std::visit( + [](auto& db) -> std::variant< + std::reference_wrapper, + std::reference_wrapper> + { return std::ref(db); }, + db + ), + prefix_data + ); if (format != QueryResultFormat::Json) { diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 9b8eabeff9..eb01231e29 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -182,7 +182,17 @@ namespace mamba } PrefixData& prefix_data = exp_prefix_data.value(); - load_installed_packages_in_database(ctx, std::get(db), prefix_data); + load_installed_packages_in_database( + ctx, + std::visit( + [](auto& db) -> std::variant< + std::reference_wrapper, + std::reference_wrapper> + { return std::ref(db); }, + db + ), + prefix_data + ); auto request = create_update_request(prefix_data, raw_update_specs, update_params); add_pins_to_request( diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 3679ef97cb..83ded0ce39 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -73,8 +73,10 @@ namespace } else if (auto* resolvo_db = std::get_if(&database)) { - // TODO: Implement for resolvo database - throw std::runtime_error("resolvo database not yet supported for self-update"); + auto candidates = resolvo_db->get_candidates( + resolvo_db->name_pool.alloc(resolvo::String(spec.name().to_string())) + ); + return !candidates.candidates.empty(); } return found; }; @@ -100,8 +102,22 @@ namespace } else if (auto* resolvo_db = std::get_if(&database)) { - // TODO: Implement for resolvo database - throw std::runtime_error("resolvo database not yet supported for self-update"); + // For resolvo, we need to get all candidates for the package and find the latest + // version + auto candidates = resolvo_db->get_candidates( + resolvo_db->name_pool.alloc(resolvo::String(spec.name().to_string())) + ); + if (candidates.candidates.empty()) + { + return std::nullopt; + } + + // Sort candidates by version + resolvo_db->sort_candidates(candidates.candidates); + + // Get the latest version (last in the sorted list) + auto latest_solvable = candidates.candidates[candidates.candidates.size() - 1]; + return resolvo_db->solvable_pool[latest_solvable]; } return out; }; @@ -129,7 +145,7 @@ update_self(Configuration& config, const std::optional& version) { solver::libsolv::Database database{ channel_context.params() }; add_spdlog_logger_to_database(database); - return std::move(database); + return database; } }(); @@ -180,19 +196,7 @@ update_self(Configuration& config, const std::optional& version) ); ctx.download_only = true; - auto t = [&]() -> MTransaction - { - if (auto* libsolv_db = std::get_if(&db_variant)) - { - return MTransaction(ctx, db_variant, { latest_micromamba.value() }, package_caches); - } - else if (auto* resolvo_db = std::get_if(&db_variant)) - { - // TODO: Implement for resolvo database - throw std::runtime_error("resolvo database not yet supported for self-update"); - } - throw std::runtime_error("Invalid database variant type"); - }(); + auto t = MTransaction(ctx, db_variant, { latest_micromamba.value() }, package_caches); auto exp_prefix_data = PrefixData::create(ctx.prefix_params.root_prefix, channel_context); if (!exp_prefix_data) { From f0487f67961d08e89257bca8ed6a2a4ccacb6fc5 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 12:29:19 +0200 Subject: [PATCH 41/59] Use resolvo 0.1.0 for now Signed-off-by: Julien Jerphanion --- .github/workflows/static_build.yml | 2 +- dev/environment-dev.yml | 2 +- dev/environment-micromamba-static.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static_build.yml b/.github/workflows/static_build.yml index 045e29af52..5d47e6ee33 100644 --- a/.github/workflows/static_build.yml +++ b/.github/workflows/static_build.yml @@ -71,7 +71,7 @@ jobs: export FEEDSTOCK_ROOT="${PWD}" export CI="local" # Patch: add resolvo-cpp as a host dependency - sed -i 's/ - fmt/ - fmt\n - resolvo-cpp/' recipe/meta.yaml + sed -i 's/ - fmt/ - fmt\n - resolvo-cpp==0.1.0/' recipe/meta.yaml # For OSX not using Docker export CONDA_BLD_PATH="${PWD}/build_artifacts" mkdir -p "${CONDA_BLD_PATH}" diff --git a/dev/environment-dev.yml b/dev/environment-dev.yml index cf714f0f2a..e1d275158b 100644 --- a/dev/environment-dev.yml +++ b/dev/environment-dev.yml @@ -15,7 +15,7 @@ dependencies: - libarchive>=3.8 lgpl_* - libcurl >=7.86 - libsodium - - resolvo-cpp + - resolvo-cpp==0.1.0 - libsolv >=0.7.18 - nlohmann_json - reproc-cpp >=14.2.4.post0 diff --git a/dev/environment-micromamba-static.yml b/dev/environment-micromamba-static.yml index 3e13582040..ad38a5a1bd 100644 --- a/dev/environment-micromamba-static.yml +++ b/dev/environment-micromamba-static.yml @@ -13,7 +13,7 @@ dependencies: - simdjson-static >=3.3.0 - spdlog - fmt >=11.1.0 - - resolvo-cpp + - resolvo-cpp==0.1.0 - libsolv-static >=0.7.24 - yaml-cpp-static >=0.8.0 - reproc-static >=14.2.4.post0 From 09761531896aef176bc73213965d73eb5b62bada Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 13:00:28 +0200 Subject: [PATCH 42/59] Pass ChannelParams to solver::resolvo::Database Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/solver/resolvo/database.hpp | 6 +++++- libmamba/src/api/install.cpp | 6 ++++-- libmamba/src/solver/resolvo/database.cpp | 8 +++++++- micromamba/src/update.cpp | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index bc20fec4ba..cc69fdadbb 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -200,9 +200,11 @@ namespace mamba::solver::resolvo { public: - Database(); + explicit Database(specs::ChannelResolveParams channel_params); ~Database() override = default; + [[nodiscard]] auto channel_params() const -> const specs::ChannelResolveParams&; + // Implementation of mamba::solver::Database interface void add_repo_from_repodata_json( const fs::u8path& filename, @@ -256,6 +258,8 @@ namespace mamba::solver::resolvo std::unordered_map<::resolvo::NameId, ::resolvo::Vector<::resolvo::SolvableId>> name_to_solvable; std::unordered_map<::resolvo::VersionSetId, std::pair> version_set_to_max_version_and_track_features_numbers; + + specs::ChannelResolveParams m_channel_params; }; } diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index f869c4a443..a0e4e69344 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -753,8 +753,10 @@ namespace mamba : solver::libsolv::MatchSpecParser::Libsolv }; solver::DatabaseVariant db = ctx.experimental_resolvo_solver - ? solver::DatabaseVariant(std::in_place_type< - solver::resolvo::Database>) + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) : solver::DatabaseVariant( std::in_place_type, channel_context.params(), diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index f1756b807e..f278b5efb9 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -195,11 +195,17 @@ namespace mamba::solver::resolvo } } - Database::Database() + Database::Database(specs::ChannelResolveParams channel_params) : name_pool(Mapping<::resolvo::NameId, ::resolvo::String>()) + , m_channel_params(std::move(channel_params)) { } + auto Database::channel_params() const -> const specs::ChannelResolveParams& + { + return m_channel_params; + } + void Database::add_repo_from_repodata_json( const fs::u8path& filename, const std::string& repo_url, diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 83ded0ce39..696b6af3e4 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -139,7 +139,7 @@ update_self(Configuration& config, const std::optional& version) { if (ctx.experimental_resolvo_solver) { - return solver::resolvo::Database{}; + return solver::resolvo::Database{ channel_context.params() }; } else { From bb06b21d0c92f53bab9407f69ccc34f887874c73 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 13:38:16 +0200 Subject: [PATCH 43/59] Revert changes made to `expected_caster.hpp` Signed-off-by: Julien Jerphanion --- .../libmambapy/bindings/expected_caster.hpp | 123 ------------------ 1 file changed, 123 deletions(-) diff --git a/libmambapy/src/libmambapy/bindings/expected_caster.hpp b/libmambapy/src/libmambapy/bindings/expected_caster.hpp index af1e329c25..b8c941c48c 100644 --- a/libmambapy/src/libmambapy/bindings/expected_caster.hpp +++ b/libmambapy/src/libmambapy/bindings/expected_caster.hpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -89,128 +88,6 @@ namespace PYBIND11_NAMESPACE } PYBIND11_TYPE_CASTER(value_type, const_name("None")); - - /** - * A caster for std::variant with reference_wrapper types. - */ - template - struct type_caster...>> - { - using value_type = std::variant...>; - - bool load(handle src, bool convert) - { - return false; // We don't support loading from Python - } - - static handle cast(const value_type& src, return_value_policy policy, handle parent) - { - return std::visit( - [policy, parent](const auto& v) -> handle - { - using T = std::decay_t; - return make_caster::cast(v.get(), policy, parent); - }, - src - ); - } - - PYBIND11_TYPE_CASTER(value_type, _("variant")); - }; - - /** - * A caster for std::reference_wrapper. - */ - template - struct type_caster> - { - using value_type = std::reference_wrapper; - - bool load(handle src, bool convert) - { - return false; // We don't support loading from Python - } - - static handle cast(const value_type& src, return_value_policy policy, handle parent) - { - return make_caster::cast(src.get(), policy, parent); - } - - PYBIND11_TYPE_CASTER(value_type, _("reference_wrapper")); - }; - - /** - * Specialization for the specific variant type used in the database. - */ - template <> - struct type_caster, - std::reference_wrapper>> - { - using value_type = std::variant< - std::reference_wrapper, - std::reference_wrapper>; - - bool load(handle src, bool convert) - { - return false; // We don't support loading from Python - } - - static handle cast(const value_type& src, return_value_policy policy, handle parent) - { - return std::visit( - [policy, parent](const auto& v) -> handle - { - using T = std::decay_t; - return make_caster::cast(v.get(), policy, parent); - }, - src - ); - } - - PYBIND11_TYPE_CASTER(value_type, _("variant")); - }; - - /** - * A caster for mamba::Context. - */ - template <> - struct type_caster - { - using value_type = mamba::Context; - - bool load(handle src, bool convert) - { - return false; // We don't support loading from Python - } - - static handle cast(const value_type& src, return_value_policy policy, handle parent) - { - return make_caster::cast(src, policy, parent); - } - - PYBIND11_TYPE_CASTER(value_type, _("Context")); - }; - - /** - * A caster for mamba::PrefixData. - */ - template <> - struct type_caster - { - using value_type = mamba::PrefixData; - - bool load(handle src, bool convert) - { - return false; // We don't support loading from Python - } - - static handle cast(const value_type& src, return_value_policy policy, handle parent) - { - return make_caster::cast(src, policy, parent); - } - - PYBIND11_TYPE_CASTER(value_type, _("PrefixData")); }; } } From b50b59aac0d3068d3f594131eb9cc59b71c57721 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 13:51:19 +0200 Subject: [PATCH 44/59] Adapt `create_repo_from_pkgs_dir` Signed-off-by: Julien Jerphanion --- libmamba/src/api/channel_loader.cpp | 77 +++++++++++++---------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index dd1e1d034d..e2917a0128 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -25,9 +25,9 @@ namespace mamba auto create_repo_from_pkgs_dir( const Context& ctx, ChannelContext& channel_context, - solver::libsolv::Database& database, + solver::DatabaseVariant& database, const fs::u8path& pkgs_dir - ) -> solver::libsolv::RepoInfo + ) -> void { if (!fs::exists(pkgs_dir)) { @@ -51,22 +51,40 @@ namespace mamba } // Create a repo from the packages - auto repo = database.add_repo_from_packages( - prefix_data.sorted_records(), - "pkgs_dir", - solver::libsolv::PipAsPythonDependency::No - ); + if (auto* libsolv_db = std::get_if(&database)) + { + libsolv_db->add_repo_from_packages( + prefix_data.sorted_records(), + "pkgs_dir", + solver::libsolv::PipAsPythonDependency::No + ); - // Load the packages into the database - load_installed_packages_in_database( - ctx, - std::variant< - std::reference_wrapper, - std::reference_wrapper>(std::ref(database)), - prefix_data - ); + // Load the packages into the database + load_installed_packages_in_database( + ctx, + std::variant< + std::reference_wrapper, + std::reference_wrapper>(std::ref(*libsolv_db)), + prefix_data + ); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + resolvo_db->add_repo_from_packages(prefix_data.sorted_records(), "pkgs_dir", false); - return repo; + // Load the packages into the database + load_installed_packages_in_database( + ctx, + std::variant< + std::reference_wrapper, + std::reference_wrapper>(std::ref(*resolvo_db)), + prefix_data + ); + } + else + { + throw std::runtime_error("Invalid database variant"); + } } void create_subdirs( @@ -276,32 +294,7 @@ namespace mamba LOG_INFO << "Creating repo from pkgs_dir for offline"; for (const auto& c : ctx.pkgs_dirs) { - if (auto* libsolv_db = std::get_if(&database)) - { - create_repo_from_pkgs_dir(ctx, channel_context, *libsolv_db, c); - } - else if (auto* resolvo_db = std::get_if(&database)) - { - // For resolvo, we need to create a vector of PackageInfo objects from the - // pkgs_dir - std::vector offline_packages; - for (const auto& entry : fs::directory_iterator(c)) - { - if (entry.path().extension() == ".tar.bz2" - || entry.path().extension() == ".conda") - { - auto pkg_info = specs::PackageInfo::from_url(entry.path().string()) - .or_else([](specs::ParseError&& err) - { throw std::move(err); }) - .value(); - offline_packages.push_back(std::move(pkg_info)); - } - } - if (!offline_packages.empty()) - { - resolvo_db->add_repo_from_packages(offline_packages, "offline_packages"); - } - } + create_repo_from_pkgs_dir(ctx, channel_context, database, c); } } std::string prev_channel; From 28fda5402ab3a4a281d5d9d899b565f5f60297ed Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 14:11:29 +0200 Subject: [PATCH 45/59] Merge `database_has_package` implementations Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/solver/database.hpp | 15 ++++++++++++ .../include/mamba/solver/libsolv/database.hpp | 14 +++++++++++ .../include/mamba/solver/resolvo/database.hpp | 7 ++++++ libmamba/src/core/transaction.cpp | 19 +-------------- micromamba/src/update.cpp | 23 ++----------------- 5 files changed, 39 insertions(+), 39 deletions(-) diff --git a/libmamba/include/mamba/solver/database.hpp b/libmamba/include/mamba/solver/database.hpp index 7037f064e4..324dedf7cc 100644 --- a/libmamba/include/mamba/solver/database.hpp +++ b/libmamba/include/mamba/solver/database.hpp @@ -36,7 +36,22 @@ namespace mamba::solver ) = 0; virtual void set_installed_repo(const std::string& repo_name) = 0; + + virtual bool has_package(const specs::MatchSpec& spec) = 0; }; + + inline auto database_has_package(DatabaseVariant& database, const specs::MatchSpec& spec) -> bool + { + if (auto* libsolv_db = std::get_if(&database)) + { + return libsolv_db->has_package(spec); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + return resolvo_db->has_package(spec); + } + throw std::runtime_error("Invalid database variant"); + } } #endif // MAMBA_SOLVER_DATABASE_HPP diff --git a/libmamba/include/mamba/solver/libsolv/database.hpp b/libmamba/include/mamba/solver/libsolv/database.hpp index 3d6cb2babb..64ac552040 100644 --- a/libmamba/include/mamba/solver/libsolv/database.hpp +++ b/libmamba/include/mamba/solver/libsolv/database.hpp @@ -136,6 +136,20 @@ namespace mamba::solver::libsolv template void for_each_package_depending_on(const specs::MatchSpec& ms, Func&&); + bool has_package(const specs::MatchSpec& spec) override + { + bool found = false; + for_each_package_matching( + spec, + [&](const auto&) + { + found = true; + return util::LoopControl::Break; + } + ); + return found; + } + /** * An access control wrapper. * diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index cc69fdadbb..9039550e53 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -252,6 +252,13 @@ namespace mamba::solver::resolvo Mapping<::resolvo::VersionSetId, specs::MatchSpec> version_set_pool; Mapping<::resolvo::SolvableId, specs::PackageInfo> solvable_pool; + bool has_package(const specs::MatchSpec& spec) override + { + auto candidates = get_candidates(name_pool.alloc(resolvo::String(spec.name().to_string())) + ); + return !candidates.candidates.empty(); + } + private: // Maps for quick lookups diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 0fb33591be..7858667450 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -54,24 +54,7 @@ namespace mamba auto database_has_package(solver::DatabaseVariant& database, const specs::MatchSpec& spec) -> bool { - bool found = false; - if (auto* libsolv_db = std::get_if(&database)) - { - libsolv_db->for_each_package_matching( - spec, - [&](const auto&) - { - found = true; - return util::LoopControl::Break; - } - ); - } - else if (auto* resolvo_db = std::get_if(&database)) - { - // TODO: Implement for resolvo database - throw std::runtime_error("Package matching not yet implemented for resolvo database"); - } - return found; + return solver::database_has_package(database, spec); } auto installed_python(const solver::DatabaseVariant& database) -> std::optional diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 696b6af3e4..9d9254a8cd 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -59,27 +59,8 @@ namespace { auto database_has_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> bool { - bool found = false; - if (auto* libsolv_db = std::get_if(&database)) - { - libsolv_db->for_each_package_matching( - spec, - [&](const auto&) - { - found = true; - return util::LoopControl::Break; - } - ); - } - else if (auto* resolvo_db = std::get_if(&database)) - { - auto candidates = resolvo_db->get_candidates( - resolvo_db->name_pool.alloc(resolvo::String(spec.name().to_string())) - ); - return !candidates.candidates.empty(); - } - return found; - }; + return solver::database_has_package(database, spec); + } auto database_latest_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> std::optional From 7a7fdc2bdb601c0f8e6804a55730f7766923018f Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 14:18:21 +0200 Subject: [PATCH 46/59] Deduplicate `database_has_package` Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/solver/database.hpp | 25 +++++++++++-------- .../include/mamba/solver/database_utils.hpp | 24 ++++++++++++++++++ .../include/mamba/solver/libsolv/database.hpp | 2 +- .../include/mamba/solver/resolvo/database.hpp | 5 ++-- libmamba/src/core/transaction.cpp | 9 ++----- micromamba/src/update.cpp | 11 ++++---- 6 files changed, 50 insertions(+), 26 deletions(-) create mode 100644 libmamba/include/mamba/solver/database_utils.hpp diff --git a/libmamba/include/mamba/solver/database.hpp b/libmamba/include/mamba/solver/database.hpp index 324dedf7cc..f6363a7a34 100644 --- a/libmamba/include/mamba/solver/database.hpp +++ b/libmamba/include/mamba/solver/database.hpp @@ -8,10 +8,12 @@ #define MAMBA_SOLVER_DATABASE_HPP #include +#include #include #include "mamba/fs/filesystem.hpp" #include "mamba/specs/channel.hpp" +#include "mamba/specs/match_spec.hpp" #include "mamba/specs/package_info.hpp" namespace mamba::solver @@ -40,18 +42,21 @@ namespace mamba::solver virtual bool has_package(const specs::MatchSpec& spec) = 0; }; - inline auto database_has_package(DatabaseVariant& database, const specs::MatchSpec& spec) -> bool + namespace libsolv { - if (auto* libsolv_db = std::get_if(&database)) - { - return libsolv_db->has_package(spec); - } - else if (auto* resolvo_db = std::get_if(&database)) - { - return resolvo_db->has_package(spec); - } - throw std::runtime_error("Invalid database variant"); + class Database; } + + namespace resolvo + { + class Database; + } + + using DatabaseVariant = std::variant; + + // Remove or comment out the inline database_has_package function if DatabaseVariant is not + // visible or causes errors inline auto database_has_package(DatabaseVariant& database, const + // specs::MatchSpec& spec) -> bool; } #endif // MAMBA_SOLVER_DATABASE_HPP diff --git a/libmamba/include/mamba/solver/database_utils.hpp b/libmamba/include/mamba/solver/database_utils.hpp new file mode 100644 index 0000000000..4a7b9986db --- /dev/null +++ b/libmamba/include/mamba/solver/database_utils.hpp @@ -0,0 +1,24 @@ +#ifndef MAMBA_SOLVER_DATABASE_UTILS_HPP +#define MAMBA_SOLVER_DATABASE_UTILS_HPP + +#include + +#include "mamba/solver/database.hpp" + +namespace mamba::solver +{ + inline bool database_has_package(DatabaseVariant& database, const specs::MatchSpec& spec) + { + if (auto* libsolv_db = std::get_if(&database)) + { + return libsolv_db->has_package(spec); + } + else if (auto* resolvo_db = std::get_if(&database)) + { + return resolvo_db->has_package(spec); + } + throw std::runtime_error("Invalid database variant"); + } +} + +#endif // MAMBA_SOLVER_DATABASE_UTILS_HPP diff --git a/libmamba/include/mamba/solver/libsolv/database.hpp b/libmamba/include/mamba/solver/libsolv/database.hpp index 64ac552040..de4e3ab477 100644 --- a/libmamba/include/mamba/solver/libsolv/database.hpp +++ b/libmamba/include/mamba/solver/libsolv/database.hpp @@ -136,7 +136,7 @@ namespace mamba::solver::libsolv template void for_each_package_depending_on(const specs::MatchSpec& ms, Func&&); - bool has_package(const specs::MatchSpec& spec) override + bool has_package(const specs::MatchSpec& spec) { bool found = false; for_each_package_matching( diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index 9039550e53..77012b37b7 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -252,9 +252,10 @@ namespace mamba::solver::resolvo Mapping<::resolvo::VersionSetId, specs::MatchSpec> version_set_pool; Mapping<::resolvo::SolvableId, specs::PackageInfo> solvable_pool; - bool has_package(const specs::MatchSpec& spec) override + bool has_package(const specs::MatchSpec& spec) { - auto candidates = get_candidates(name_pool.alloc(resolvo::String(spec.name().to_string())) + auto candidates = get_candidates( + name_pool.alloc(::resolvo::String(spec.name().to_string())) ); return !candidates.candidates.empty(); } diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 7858667450..40ac4de51c 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -28,6 +28,7 @@ #include "mamba/core/thread_utils.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util_os.hpp" +#include "mamba/solver/database_utils.hpp" #include "mamba/solver/libsolv/database.hpp" #include "mamba/specs/match_spec.hpp" #include "mamba/util/environment.hpp" @@ -51,12 +52,6 @@ namespace mamba && caches.get_tarball_path(pkg_info).empty(); } - auto database_has_package(solver::DatabaseVariant& database, const specs::MatchSpec& spec) - -> bool - { - return solver::database_has_package(database, spec); - } - auto installed_python(const solver::DatabaseVariant& database) -> std::optional { if (auto* libsolv_db = std::get_if(&database)) @@ -149,7 +144,7 @@ namespace mamba for (const auto& pkg : pkgs_to_remove) { auto spec = explicit_spec(pkg); - if (!database_has_package(database, spec)) + if (!mamba::solver::database_has_package(database, spec)) { not_found << "\n - " << spec.to_string(); } diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 9d9254a8cd..0b4e3283ae 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -16,6 +16,7 @@ #include "mamba/core/package_database_loader.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util_os.hpp" +#include "mamba/solver/database_utils.hpp" #include "mamba/solver/solver_factory.hpp" #include "mamba/util/build.hpp" @@ -57,11 +58,6 @@ set_update_command(CLI::App* subcom, Configuration& config) #ifdef BUILDING_MICROMAMBA namespace { - auto database_has_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> bool - { - return solver::database_has_package(database, spec); - } - auto database_latest_package(solver::DatabaseVariant& database, specs::MatchSpec spec) -> std::optional { @@ -149,7 +145,10 @@ update_self(Configuration& config, const std::optional& version) if (!latest_micromamba.has_value()) { - if (database_has_package(db_variant, specs::MatchSpec::parse("micromamba").value())) + if (mamba::solver::database_has_package( + db_variant, + specs::MatchSpec::parse("micromamba").value() + )) { Console::instance().print( fmt::format("\nYour micromamba version ({}) is already up to date.", umamba::version()) From 413dd8257fc47d5baef618200fa290206d74fecb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 14:41:41 +0200 Subject: [PATCH 47/59] Resolvo on the install path Signed-off-by: Julien Jerphanion --- .../mamba/core/package_database_loader.hpp | 4 +- .../include/mamba/solver/resolvo/database.hpp | 2 + libmamba/src/api/install.cpp | 39 +++---- libmamba/src/api/remove.cpp | 36 ++++-- libmamba/src/api/repoquery.cpp | 33 ++++-- libmamba/src/api/update.cpp | 79 ++++++++----- libmamba/src/core/package_database_loader.cpp | 109 +++++++++++++----- 7 files changed, 195 insertions(+), 107 deletions(-) diff --git a/libmamba/include/mamba/core/package_database_loader.hpp b/libmamba/include/mamba/core/package_database_loader.hpp index 50fe26e60f..1b6859d67e 100644 --- a/libmamba/include/mamba/core/package_database_loader.hpp +++ b/libmamba/include/mamba/core/package_database_loader.hpp @@ -32,7 +32,9 @@ namespace mamba auto load_subdir_in_database( // const Context& ctx, - solver::libsolv::Database& database, + std::variant< + std::reference_wrapper, + std::reference_wrapper> database, const SubdirIndexLoader& subdir ) -> expected_t; diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index 77012b37b7..ffc00a310c 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -16,6 +16,8 @@ #include #include "mamba/solver/database.hpp" +#include "mamba/solver/libsolv/parameters.hpp" +#include "mamba/solver/libsolv/repo_info.hpp" #include "mamba/specs/match_spec.hpp" #include "mamba/specs/package_info.hpp" #include "mamba/specs/version.hpp" diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index a0e4e69344..fee6bbac2b 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -554,18 +554,22 @@ namespace mamba LOG_WARNING << "No 'channels' specified"; } - const std::vector matchspec_parsers{ - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv - }; - solver::libsolv::Database libsolv_db( + solver::libsolv::Database initial_db( channel_context.params(), { ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba : solver::libsolv::MatchSpecParser::Libsolv } ); - solver::DatabaseVariant db_variant = std::move(libsolv_db); + solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant(std::move(initial_db)); - add_spdlog_logger_to_database(std::get(db_variant)); + if (!ctx.experimental_resolvo_solver) + { + add_spdlog_logger_to_database(std::get(db_variant)); + } auto maybe_load = load_channels(ctx, channel_context, db_variant, package_caches); if (!maybe_load) @@ -580,24 +584,17 @@ namespace mamba } PrefixData& prefix_data = maybe_prefix_data.value(); - if (auto* libsolv_db = std::get_if(&db_variant)) + if (auto* libsolv_db_ptr = std::get_if(&db_variant)) { - load_installed_packages_in_database(ctx, *libsolv_db, prefix_data); + load_installed_packages_in_database(ctx, *libsolv_db_ptr, prefix_data); } - else if (auto* resolvo_db = std::get_if(&db_variant)) + else if (auto* resolvo_db_ptr = std::get_if(&db_variant)) { - load_installed_packages_in_database(ctx, *resolvo_db, prefix_data); + load_installed_packages_in_database(ctx, *resolvo_db_ptr, prefix_data); } auto request = create_install_request(prefix_data, raw_specs, freeze_installed); - add_pins_to_request( - request, - ctx, - prefix_data, - raw_specs, - /* no_pin= */ config.at("no_pin").value(), - /* no_py_pin = */ config.at("no_py_pin").value() - ); + add_pins_to_request(request, ctx, prefix_data, raw_specs, no_pin, no_py_pin); request.flags = ctx.solver_flags; @@ -748,10 +745,6 @@ namespace mamba bool remove_prefix_on_failure ) { - const std::vector matchspec_parsers2{ - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv - }; solver::DatabaseVariant db = ctx.experimental_resolvo_solver ? solver::DatabaseVariant( std::in_place_type, diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index d753a2e682..925a6d48f3 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -143,13 +143,27 @@ namespace mamba : solver::libsolv::MatchSpecParser::Libsolv, }, }; - add_spdlog_logger_to_database(database); + solver::DatabaseVariant db = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant(std::move(database)); + + if (!ctx.experimental_resolvo_solver) + { + add_spdlog_logger_to_database(std::get(db)); + } load_installed_packages_in_database( ctx, - std::variant< - std::reference_wrapper, - std::reference_wrapper>(std::ref(database)), + std::visit( + [](auto& db) -> std::variant< + std::reference_wrapper, + std::reference_wrapper> + { return std::ref(db); }, + db + ), prefix_data ); @@ -188,7 +202,7 @@ namespace mamba pkgs_to_remove.push_back(iter->second); } } - solver::DatabaseVariant db_variant = std::move(database); + solver::DatabaseVariant db_variant = std::move(db); auto transaction = MTransaction(ctx, db_variant, pkgs_to_remove, {}, package_caches); return execute_transaction(transaction); } @@ -206,7 +220,7 @@ namespace mamba auto outcome = solver::libsolv::Solver() .solve( - database, + std::get(db), request, ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba @@ -217,9 +231,11 @@ namespace mamba { if (ctx.output_params.json) { - Console::instance().json_write({ { "success", false }, - { "solver_problems", - unsolvable->problems(database) } }); + Console::instance().json_write( + { { "success", false }, + { "solver_problems", + unsolvable->problems(std::get(db)) } } + ); } throw mamba_error( "Could not solve for environment specs", @@ -228,7 +244,7 @@ namespace mamba } Console::instance().json_write({ { "success", true } }); - solver::DatabaseVariant db_variant2 = std::move(database); + solver::DatabaseVariant db_variant2 = std::move(db); auto transaction2 = MTransaction( ctx, db_variant2, diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index 00da72ff79..427338d8ce 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -33,14 +33,25 @@ namespace mamba config.load(); auto channel_context = ChannelContext::make_conda_compatible(ctx); - solver::DatabaseVariant db( - std::in_place_type, - channel_context.params(), - solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } - ); - add_spdlog_logger_to_database(std::get(db)); + + solver::DatabaseVariant database = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); + + if (!ctx.experimental_resolvo_solver) + { + add_spdlog_logger_to_database(std::get(database)); + } // bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds); MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); @@ -65,7 +76,7 @@ namespace mamba std::reference_wrapper, std::reference_wrapper> { return std::ref(db); }, - db + database ), prefix_data ); @@ -82,13 +93,13 @@ namespace mamba { Console::stream() << "Getting repodata from channels..." << std::endl; } - auto exp_load = load_channels(ctx, channel_context, db, package_caches); + auto exp_load = load_channels(ctx, channel_context, database, package_caches); if (!exp_load) { throw std::runtime_error(exp_load.error().what()); } } - return db; + return database; } auto make_repoquery( diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index eb01231e29..0efa36a249 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -157,14 +157,23 @@ namespace mamba populate_context_channels_from_specs(raw_update_specs, ctx); - solver::DatabaseVariant db( - std::in_place_type, - channel_context.params(), - solver::libsolv::Database::Settings{ ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } - ); - add_spdlog_logger_to_database(std::get(db)); + solver::DatabaseVariant db = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant(solver::libsolv::Database{ + channel_context.params(), + { + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv, + } }); + + if (!ctx.experimental_resolvo_solver) + { + add_spdlog_logger_to_database(std::get(db)); + } MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); @@ -212,16 +221,31 @@ namespace mamba // Console stream prints on destruction } - auto outcome = solver::libsolv::Solver() - .solve( - std::get(db), - request, - ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Mixed - ) - .value(); - if (auto* unsolvable = std::get_if(&outcome)) + using LibsolvOutcome = std::variant; + auto outcome = ctx.experimental_resolvo_solver + ? solver::resolvo::Solver() + .solve(std::get(db), request) + .map( + [](auto&& result) -> LibsolvOutcome + { + // resolvo only returns Solution + return std::get(result); + } + ) + : solver::libsolv::Solver().solve( + std::get(db), + request, + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv + ); + + if (!outcome.has_value()) + { + throw std::runtime_error(outcome.error().what()); + } + auto& result = outcome.value(); + if (auto* unsolvable = std::get_if(&result)) { unsolvable->explain_problems_to( std::get(db), @@ -233,11 +257,10 @@ namespace mamba ); if (ctx.output_params.json) { - Console::instance().json_write( - { { "success", false }, - { "solver_problems", - unsolvable->problems(std::get(db)) } } - ); + Console::instance().json_write(nlohmann::json{ + { "success", false }, + { "solver_problems", + unsolvable->problems(std::get(db)) } }); } throw mamba_error( "Could not solve for environment specs", @@ -245,14 +268,8 @@ namespace mamba ); } - Console::instance().json_write({ { "success", true } }); - auto transaction = MTransaction( - ctx, - db, - request, - std::get(outcome), - package_caches - ); + Console::instance().json_write(nlohmann::json{ { "success", true } }); + auto transaction = MTransaction(ctx, db, request, std::get(result), package_caches); auto execute_transaction = [&](MTransaction& trans) diff --git a/libmamba/src/core/package_database_loader.cpp b/libmamba/src/core/package_database_loader.cpp index 581e378194..9228944037 100644 --- a/libmamba/src/core/package_database_loader.cpp +++ b/libmamba/src/core/package_database_loader.cpp @@ -55,7 +55,9 @@ namespace mamba auto load_subdir_in_database( const Context& ctx, - solver::libsolv::Database& database, + std::variant< + std::reference_wrapper, + std::reference_wrapper> database, const SubdirIndexLoader& subdir ) -> expected_t { @@ -78,11 +80,28 @@ namespace mamba auto maybe_repo = subdir.valid_libsolv_cache_path().and_then( [&](fs::u8path&& solv_file) { - return database.add_repo_from_native_serialization( - solv_file, - expected_cache_origin, - subdir.channel_id(), - add_pip + return std::visit( + [&](auto& db) -> expected_t + { + using DB = std::decay_t; + if constexpr (std::is_same_v>) + { + return db.get().add_repo_from_native_serialization( + solv_file, + expected_cache_origin, + subdir.channel_id(), + add_pip + ); + } + else + { + return make_unexpected( + "Native serialization not supported for resolvo::Database", + mamba_error_code::unknown + ); + } + }, + database ); } ); @@ -97,18 +116,35 @@ namespace mamba [&](fs::u8path&& repodata_json) { using PackageTypes = solver::libsolv::PackageTypes; - LOG_INFO << "Trying to load repo from json file " << repodata_json; - return database.add_repo_from_repodata_json( - repodata_json, - util::rsplit(subdir.metadata().url(), "/", 1).front(), - subdir.channel_id(), - add_pip, - ctx.use_only_tar_bz2 ? PackageTypes::TarBz2Only - : PackageTypes::CondaOrElseTarBz2, - static_cast(ctx.validation_params.verify_artifacts - ), - json_parser + return std::visit( + [&](auto& db) -> expected_t + { + using DB = std::decay_t; + if constexpr (std::is_same_v>) + { + return db.get().add_repo_from_repodata_json( + repodata_json, + util::rsplit(subdir.metadata().url(), "/", 1).front(), + subdir.channel_id(), + add_pip, + ctx.use_only_tar_bz2 ? PackageTypes::TarBz2Only + : PackageTypes::CondaOrElseTarBz2, + static_cast( + ctx.validation_params.verify_artifacts + ), + json_parser + ); + } + else + { + return make_unexpected( + "add_repo_from_repodata_json not supported for resolvo::Database", + mamba_error_code::unknown + ); + } + }, + database ); } ) @@ -117,22 +153,33 @@ namespace mamba { if (!util::on_win) { - database - .native_serialize_repo( - repo, - subdir.writable_libsolv_cache_path(), - expected_cache_origin - ) - .or_else( - [&](const auto& err) + std::visit( + [&](auto& db) + { + using DB = std::decay_t; + if constexpr (std::is_same_v>) { - LOG_WARNING << R"(Fail to write native serialization to file ")" - << subdir.writable_libsolv_cache_path() - << R"(" for repo ")" << subdir.name() << ": " - << err.what(); - ; + db.get() + .native_serialize_repo( + repo, + subdir.writable_libsolv_cache_path(), + expected_cache_origin + ) + .or_else( + [&](const auto& err) + { + LOG_WARNING + << R"(Fail to write native serialization to file ")" + << subdir.writable_libsolv_cache_path() + << R"(" for repo ")" << subdir.name() << ": " + << err.what(); + } + ); } - ); + // else: do nothing for resolvo::Database + }, + database + ); } return std::move(repo); } From 4bce239e072ec742eaa7bd1e1c71341f8cce2063 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 15:11:32 +0200 Subject: [PATCH 48/59] More install path Signed-off-by: Julien Jerphanion --- .../mamba/core/package_database_loader.hpp | 2 +- libmamba/src/api/channel_loader.cpp | 47 +++++++-------- libmamba/src/core/package_database_loader.cpp | 57 ++++++------------- .../tests/src/solver/test_problems_graph.cpp | 2 +- libmambapy/src/libmambapy/bindings/legacy.cpp | 19 ++++++- 5 files changed, 55 insertions(+), 72 deletions(-) diff --git a/libmamba/include/mamba/core/package_database_loader.hpp b/libmamba/include/mamba/core/package_database_loader.hpp index 1b6859d67e..e952c5b255 100644 --- a/libmamba/include/mamba/core/package_database_loader.hpp +++ b/libmamba/include/mamba/core/package_database_loader.hpp @@ -36,7 +36,7 @@ namespace mamba std::reference_wrapper, std::reference_wrapper> database, const SubdirIndexLoader& subdir - ) -> expected_t; + ) -> expected_t; auto load_installed_packages_in_database( const Context& ctx, diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index e2917a0128..147bb762b8 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -316,32 +316,27 @@ namespace mamba if (auto* libsolv_db = std::get_if(&database)) { - load_subdir_in_database(ctx, *libsolv_db, subdir) - .transform([&](solver::libsolv::RepoInfo&& repo) - { libsolv_db->set_repo_priority(repo, priorities[i]); }) - .or_else( - [&](const auto&) - { - if (is_retry) - { - std::stringstream ss; - ss << "Could not load repodata.json for " << subdir.name() - << " after retry." - << "Please check repodata source. Exiting." << std::endl; - error_list.push_back( - mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) - ); - } - else - { - LOG_WARNING << "Could not load repodata.json for " - << subdir.name() - << ". Deleting cache, and retrying."; - subdir.clear_valid_cache_files(); - loading_failed = true; - } - } - ); + auto res = load_subdir_in_database(ctx, *libsolv_db, subdir); + if (!res) + { + if (is_retry) + { + std::stringstream ss; + ss << "Could not load repodata.json for " << subdir.name() + << " after retry." + << "Please check repodata source. Exiting." << std::endl; + error_list.push_back( + mamba_error(ss.str(), mamba_error_code::repodata_not_loaded) + ); + } + else + { + LOG_WARNING << "Could not load repodata.json for " << subdir.name() + << ". Deleting cache, and retrying."; + subdir.clear_valid_cache_files(); + loading_failed = true; + } + } } else if (auto* resolvo_db = std::get_if(&database)) { diff --git a/libmamba/src/core/package_database_loader.cpp b/libmamba/src/core/package_database_loader.cpp index 9228944037..8fc0ab9e9b 100644 --- a/libmamba/src/core/package_database_loader.cpp +++ b/libmamba/src/core/package_database_loader.cpp @@ -59,7 +59,7 @@ namespace mamba std::reference_wrapper, std::reference_wrapper> database, const SubdirIndexLoader& subdir - ) -> expected_t + ) -> expected_t { const auto expected_cache_origin = solver::libsolv::RepodataOrigin{ /* .url= */ util::rsplit(subdir.metadata().url(), "/", 1).front(), @@ -81,17 +81,18 @@ namespace mamba [&](fs::u8path&& solv_file) { return std::visit( - [&](auto& db) -> expected_t + [&](auto& db) -> expected_t { using DB = std::decay_t; if constexpr (std::is_same_v>) { - return db.get().add_repo_from_native_serialization( + db.get().add_repo_from_native_serialization( solv_file, expected_cache_origin, subdir.channel_id(), add_pip ); + return {}; } else { @@ -107,7 +108,7 @@ namespace mamba ); if (maybe_repo) { - return maybe_repo; + return {}; } } @@ -118,12 +119,12 @@ namespace mamba using PackageTypes = solver::libsolv::PackageTypes; LOG_INFO << "Trying to load repo from json file " << repodata_json; return std::visit( - [&](auto& db) -> expected_t + [&](auto& db) -> expected_t { using DB = std::decay_t; if constexpr (std::is_same_v>) { - return db.get().add_repo_from_repodata_json( + db.get().add_repo_from_repodata_json( repodata_json, util::rsplit(subdir.metadata().url(), "/", 1).front(), subdir.channel_id(), @@ -135,13 +136,17 @@ namespace mamba ), json_parser ); + return {}; } else { - return make_unexpected( - "add_repo_from_repodata_json not supported for resolvo::Database", - mamba_error_code::unknown + db.get().add_repo_from_repodata_json( + repodata_json, + util::rsplit(subdir.metadata().url(), "/", 1).front(), + subdir.channel_id(), + false ); + return {}; } }, database @@ -149,39 +154,9 @@ namespace mamba } ) .transform( - [&](solver::libsolv::RepoInfo&& repo) -> solver::libsolv::RepoInfo + [&](void) -> void { - if (!util::on_win) - { - std::visit( - [&](auto& db) - { - using DB = std::decay_t; - if constexpr (std::is_same_v>) - { - db.get() - .native_serialize_repo( - repo, - subdir.writable_libsolv_cache_path(), - expected_cache_origin - ) - .or_else( - [&](const auto& err) - { - LOG_WARNING - << R"(Fail to write native serialization to file ")" - << subdir.writable_libsolv_cache_path() - << R"(" for repo ")" << subdir.name() << ": " - << err.what(); - } - ); - } - // else: do nothing for resolvo::Database - }, - database - ); - } - return std::move(repo); + // Serialization step removed: no RepoInfo available to serialize. } ); } diff --git a/libmamba/tests/src/solver/test_problems_graph.cpp b/libmamba/tests/src/solver/test_problems_graph.cpp index 4e20c60a97..f1d3b8b9da 100644 --- a/libmamba/tests/src/solver/test_problems_graph.cpp +++ b/libmamba/tests/src/solver/test_problems_graph.cpp @@ -362,7 +362,7 @@ namespace for (auto& sub_dir : sub_dirs) { - auto repo = load_subdir_in_database(ctx, database, sub_dir); + REQUIRE(load_subdir_in_database(ctx, database, sub_dir).has_value()); } } diff --git a/libmambapy/src/libmambapy/bindings/legacy.cpp b/libmambapy/src/libmambapy/bindings/legacy.cpp index a7234e6cca..206460b33e 100644 --- a/libmambapy/src/libmambapy/bindings/legacy.cpp +++ b/libmambapy/src/libmambapy/bindings/legacy.cpp @@ -524,7 +524,15 @@ bind_submodule_impl(pybind11::module_ m) m.def( "load_subdir_in_database", - &load_subdir_in_database, + [](Context& context, auto& database, SubdirIndexLoader& subdir) + { + auto res = load_subdir_in_database(context, database, subdir); + if (!res) + { + throw std::runtime_error(res.error().what()); + } + return py::none(); + }, py::arg("context"), py::arg("database"), py::arg("subdir") @@ -738,10 +746,15 @@ bind_submodule_impl(pybind11::module_ m) .def( "create_repo", [](SubdirDataMigrator& self, Context& context, solver::libsolv::Database& database - ) -> solver::libsolv::RepoInfo + ) -> py::object { deprecated("Use libmambapy.load_subdir_in_database instead", "2.0"); - return extract(load_subdir_in_database(context, database, *self.p_subdir_index)); + auto res = load_subdir_in_database(context, database, *self.p_subdir_index); + if (!res) + { + throw std::runtime_error(res.error().what()); + } + return py::none(); }, py::arg("context"), py::arg("db") From cac9ad6334781757df015d9cc1d305a2c57432bf Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 16:16:51 +0200 Subject: [PATCH 49/59] Define `lsplit_track_features` inline Signed-off-by: Julien Jerphanion --- libmamba/src/solver/libsolv/helpers.cpp | 40 +++++++------- libmamba/src/solver/resolvo/database.cpp | 52 +++++++++---------- .../tests/src/solver/resolvo/test_solver.cpp | 27 +++++----- 3 files changed, 59 insertions(+), 60 deletions(-) diff --git a/libmamba/src/solver/libsolv/helpers.cpp b/libmamba/src/solver/libsolv/helpers.cpp index 0078c21f68..aed7351467 100644 --- a/libmamba/src/solver/libsolv/helpers.cpp +++ b/libmamba/src/solver/libsolv/helpers.cpp @@ -149,12 +149,6 @@ namespace mamba::solver::libsolv namespace { - auto lsplit_track_features(std::string_view features) - { - constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; - auto [_, tail] = util::lstrip_if_parts(features, is_sep); - return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); - } void set_solv_signatures( solv::ObjSolvableView solv, @@ -364,26 +358,34 @@ namespace mamba::solver::libsolv } } - if (auto obj = pkg["track_features"]; !obj.error()) + if (auto track_features = pkg["track_features"]; !track_features.error()) { - if (obj.is_string()) + if (auto track_features_arr = track_features.get_array(); !track_features_arr.error()) { - auto splits = lsplit_track_features(obj.get_string().value_unsafe()); - while (!splits[0].empty()) + for (auto elem : track_features_arr) { - solv.add_track_feature(splits[0]); - splits = lsplit_track_features(splits[1]); + if (auto feat = elem.get_string(); !feat.error()) + { + solv.add_track_feature(feat.value()); + } } } - else + else if (auto track_features_str = track_features.get_string(); + !track_features_str.error()) { - // assuming obj is an array - for (auto elem : obj.get_array()) + const auto lsplit_track_features = [](std::string_view features) { - if (!elem.error() && elem.is_string()) - { - solv.add_track_feature(elem.get_string().value_unsafe()); - } + constexpr auto is_sep = [](char c) -> bool + { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); + }; + + auto splits = lsplit_track_features(track_features_str.value()); + while (!splits[0].empty()) + { + solv.add_track_feature(splits[0]); + splits = lsplit_track_features(splits[1]); } } } diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index f278b5efb9..4082839d99 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -18,19 +18,9 @@ namespace mamba::solver::resolvo { namespace { - // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` - auto lsplit_track_features(std::string_view features) - { - constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; - auto [_, tail] = util::lstrip_if_parts(features, is_sep); - return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); - } - - // TODO: factorise with the implementation from `set_solvable` in - // `mamba/solver/libsolv/helpers.cpp` bool parse_packageinfo_json( const std::string_view& filename, - const simdjson::dom::element& pkg, + simdjson::ondemand::object& pkg, const specs::CondaURL& repo_url, const std::string& channel_id, Database& database @@ -44,7 +34,7 @@ namespace mamba::solver::resolvo if (auto fn = pkg["fn"].get_string(); !fn.error()) { - package_info.name = fn.value_unsafe(); + package_info.name = fn.value(); } else { @@ -54,7 +44,7 @@ namespace mamba::solver::resolvo if (auto name = pkg["name"].get_string(); !name.error()) { - package_info.name = name.value_unsafe(); + package_info.name = name.value(); } else { @@ -64,7 +54,7 @@ namespace mamba::solver::resolvo if (auto version = pkg["version"].get_string(); !version.error()) { - package_info.version = version.value_unsafe(); + package_info.version = version.value(); } else { @@ -74,7 +64,7 @@ namespace mamba::solver::resolvo if (auto build_string = pkg["build"].get_string(); !build_string.error()) { - package_info.build_string = build_string.value_unsafe(); + package_info.build_string = build_string.value(); } else { @@ -84,7 +74,7 @@ namespace mamba::solver::resolvo if (auto build_number = pkg["build_number"].get_uint64(); !build_number.error()) { - package_info.build_number = build_number.value_unsafe(); + package_info.build_number = build_number.value(); } else { @@ -94,7 +84,7 @@ namespace mamba::solver::resolvo if (auto subdir = pkg["subdir"].get_c_str(); !subdir.error()) { - package_info.platform = subdir.value_unsafe(); + package_info.platform = subdir.value(); } else { @@ -103,23 +93,23 @@ namespace mamba::solver::resolvo if (auto size = pkg["size"].get_uint64(); !size.error()) { - package_info.size = size.value_unsafe(); + package_info.size = size.value(); } if (auto md5 = pkg["md5"].get_c_str(); !md5.error()) { - package_info.md5 = md5.value_unsafe(); + package_info.md5 = md5.value(); } if (auto sha256 = pkg["sha256"].get_c_str(); !sha256.error()) { - package_info.sha256 = sha256.value_unsafe(); + package_info.sha256 = sha256.value(); } if (auto elem = pkg["noarch"]; !elem.error()) { // TODO: is the following right? - if (auto val = elem.get_bool(); !val.error() && val.value_unsafe()) + if (auto val = elem.get_bool(); !val.error() && val.value()) { package_info.noarch = specs::NoArchType::No; } @@ -131,7 +121,7 @@ namespace mamba::solver::resolvo if (auto license = pkg["license"].get_c_str(); !license.error()) { - package_info.license = license.value_unsafe(); + package_info.license = license.value(); } // TODO conda timestamp are not Unix timestamp. @@ -139,7 +129,7 @@ namespace mamba::solver::resolvo // package may get arbitrary priority. if (auto timestamp = pkg["timestamp"].get_uint64(); !timestamp.error()) { - const auto time = timestamp.value_unsafe(); + const auto time = timestamp.value(); // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; @@ -151,7 +141,7 @@ namespace mamba::solver::resolvo { if (auto dep = elem.get_c_str(); !dep.error()) { - package_info.dependencies.emplace_back(dep.value_unsafe()); + package_info.dependencies.emplace_back(dep.value()); } } } @@ -162,7 +152,7 @@ namespace mamba::solver::resolvo { if (auto cons = elem.get_c_str(); !cons.error()) { - package_info.constrains.emplace_back(cons.value_unsafe()); + package_info.constrains.emplace_back(cons.value()); } } } @@ -175,13 +165,21 @@ namespace mamba::solver::resolvo { if (auto feat = elem.get_string(); !feat.error()) { - package_info.track_features.emplace_back(feat.value_unsafe()); + package_info.track_features.emplace_back(feat.value()); } } } else if (auto track_features_str = obj.get_string(); !track_features_str.error()) { - auto splits = lsplit_track_features(track_features_str.value_unsafe()); + const auto lsplit_track_features = [](std::string_view features) + { + constexpr auto is_sep = [](char c) -> bool + { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); + }; + + auto splits = lsplit_track_features(track_features_str.value()); while (!splits[0].empty()) { package_info.track_features.emplace_back(splits[0]); diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index fc3bd9e64a..9d141f4deb 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -708,15 +708,6 @@ struct PackageDatabase : public DependencyProvider } }; -// TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` -auto -lsplit_track_features(std::string_view features) -{ - constexpr auto is_sep = [](char c) -> bool { return (c == ',') || util::is_space(c); }; - auto [_, tail] = util::lstrip_if_parts(features, is_sep); - return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); -} - // TODO: factorise with the implementation from `set_solvable` in `mamba/solver/libsolv/helpers.cpp` bool parse_packageinfo_json( @@ -858,21 +849,29 @@ parse_packageinfo_json( } } - if (auto obj = pkg["track_features"]; !obj.error()) + if (auto track_features = pkg["track_features"]; !track_features.error()) { - if (auto track_features_arr = obj.get_array(); !track_features_arr.error()) + if (auto track_features_arr = track_features.get_array(); !track_features_arr.error()) { for (auto elem : track_features_arr) { if (auto feat = elem.get_string(); !feat.error()) { - package_info.track_features.emplace_back(feat.value_unsafe()); + package_info.track_features.emplace_back(feat.value()); } } } - else if (auto track_features_str = obj.get_string(); !track_features_str.error()) + else if (auto track_features_str = track_features.get_string(); !track_features_str.error()) { - auto splits = lsplit_track_features(track_features_str.value_unsafe()); + const auto lsplit_track_features = [](std::string_view features) + { + constexpr auto is_sep = [](char c) -> bool + { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); + }; + + auto splits = lsplit_track_features(track_features_str.value()); while (!splits[0].empty()) { package_info.track_features.emplace_back(splits[0]); From 3e00e99c3252e564ffccc0934937dc5b29562ea7 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 20 May 2025 16:52:26 +0200 Subject: [PATCH 50/59] Parse respodata.json Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/specs/package_info.hpp | 10 + libmamba/src/solver/resolvo/database.cpp | 474 +++++------------- libmamba/src/specs/package_info.cpp | 171 +++++++ .../tests/src/solver/resolvo/test_solver.cpp | 229 ++------- 4 files changed, 362 insertions(+), 522 deletions(-) diff --git a/libmamba/include/mamba/specs/package_info.hpp b/libmamba/include/mamba/specs/package_info.hpp index c813f729cf..a8e7798aba 100644 --- a/libmamba/include/mamba/specs/package_info.hpp +++ b/libmamba/include/mamba/specs/package_info.hpp @@ -12,6 +12,7 @@ #include #include +#include #include "mamba/specs/error.hpp" #include "mamba/specs/platform.hpp" @@ -19,6 +20,8 @@ namespace mamba::specs { + class CondaURL; + enum class PackageType { Unknown, @@ -66,6 +69,13 @@ namespace mamba::specs [[nodiscard]] static auto from_url(std::string_view url) -> expected_parse_t; + [[nodiscard]] static auto from_json( + const std::string_view& filename, + simdjson::ondemand::object& pkg, + const CondaURL& repo_url, + const std::string& channel_id + ) -> expected_parse_t; + PackageInfo() = default; explicit PackageInfo(std::string name); PackageInfo(std::string name, std::string version, std::string build_string, std::size_t build_number); diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index 4082839d99..15ac5d8016 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -16,253 +16,180 @@ namespace mamba::solver::resolvo { - namespace - { - bool parse_packageinfo_json( - const std::string_view& filename, - simdjson::ondemand::object& pkg, - const specs::CondaURL& repo_url, - const std::string& channel_id, - Database& database - ) - { - specs::PackageInfo package_info; - - package_info.channel = channel_id; - package_info.filename = filename; - package_info.package_url = (repo_url / filename).str(specs::CondaURL::Credentials::Show); - if (auto fn = pkg["fn"].get_string(); !fn.error()) - { - package_info.name = fn.value(); - } - else - { - // Fallback from key entry - package_info.name = filename; - } + Database::Database(specs::ChannelResolveParams channel_params) + : name_pool(Mapping<::resolvo::NameId, ::resolvo::String>()) + , m_channel_params(std::move(channel_params)) + { + } - if (auto name = pkg["name"].get_string(); !name.error()) - { - package_info.name = name.value(); - } - else - { - LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")"; - return false; - } + auto Database::channel_params() const -> const specs::ChannelResolveParams& + { + return m_channel_params; + } - if (auto version = pkg["version"].get_string(); !version.error()) - { - package_info.version = version.value(); - } - else - { - LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")"; - return false; - } + void Database::add_repo_from_repodata_json( + const fs::u8path& filename, + const std::string& repo_url, + const std::string& channel_id, + bool verify_artifacts + ) + { + // BEWARE: + // We use below `simdjson`'s "on-demand" parser, which does not tolerate reading the same + // value more than once. This means we need to make sure that the objects and their fields + // are read and/or concretized only once and if we need to use them more than once we need + // to persist them in local memory. This is why the code below tries hard to pre-read the + // data needed in several parts of the computing in a way that prevents jumping up and down + // the hierarchy of json objects. When this rule is not followed, the parsing might end + // earlier than expected or might skip data that are read when they shouldn't be, leading to + // *runtime issues* that might not be visible at first. Because of these reasons, be careful + // when modifying the following parsing code. + + auto parser = simdjson::ondemand::parser(); + const auto lock = LockFile(filename); - if (auto build_string = pkg["build"].get_string(); !build_string.error()) - { - package_info.build_string = build_string.value(); - } - else - { - LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")"; - return false; - } + // The json storage must be kept alive as long as we are reading the json data. + const auto json_content = simdjson::padded_string::load(filename.string()); - if (auto build_number = pkg["build_number"].get_uint64(); !build_number.error()) - { - package_info.build_number = build_number.value(); - } - else - { - LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")"; - return false; - } + // Note that with the "on-demand" parser, documents/values/objects act as iterators + // to go through the document. + auto repodata_doc = parser.iterate(json_content); - if (auto subdir = pkg["subdir"].get_c_str(); !subdir.error()) + const auto repodata_version = [&] + { + if (auto version = repodata_doc["repodata_version"].get_int64(); !version.error()) { - package_info.platform = subdir.value(); + return version.value(); } else { - LOG_WARNING << R"(Found invalid subdir in ")" << filename << R"(")"; + return std::int64_t{ 1 }; } + }(); - if (auto size = pkg["size"].get_uint64(); !size.error()) + auto repodata_info = [&] + { + if (auto value = repodata_doc["info"]; !value.error()) { - package_info.size = size.value(); + if (auto object = value.get_object(); !object.error()) + { + return std::make_optional(object); + } } + return decltype(std::make_optional(repodata_doc["info"].get_object())){}; + }(); - if (auto md5 = pkg["md5"].get_c_str(); !md5.error()) + // An override for missing package subdir could be found at the top level + const auto default_subdir = [&] + { + if (repodata_info) { - package_info.md5 = md5.value(); + if (auto subdir = repodata_info.value()["subdir"]; !subdir.error()) + { + return std::string(subdir.get_string().value_unsafe()); + } } - if (auto sha256 = pkg["sha256"].get_c_str(); !sha256.error()) - { - package_info.sha256 = sha256.value(); - } + return std::string{}; + }(); - if (auto elem = pkg["noarch"]; !elem.error()) + // Get `base_url` in case 'repodata_version': 2 + // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md + const auto base_url = [&] + { + if (repodata_version == 2 && repodata_info) { - // TODO: is the following right? - if (auto val = elem.get_bool(); !val.error() && val.value()) + if (auto url = repodata_info.value()["base_url"]; !url.error()) { - package_info.noarch = specs::NoArchType::No; - } - else if (auto noarch = elem.get_c_str(); !noarch.error()) - { - package_info.noarch = specs::NoArchType::No; + return std::string(url.get_string().value_unsafe()); } } - if (auto license = pkg["license"].get_c_str(); !license.error()) - { - package_info.license = license.value(); - } + return repo_url; + }(); - // TODO conda timestamp are not Unix timestamp. - // Libsolv normalize them this way, we need to do the same here otherwise the current - // package may get arbitrary priority. - if (auto timestamp = pkg["timestamp"].get_uint64(); !timestamp.error()) - { - const auto time = timestamp.value(); - // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` - constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; - package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; - } + const auto parsed_url = specs::CondaURL::parse(base_url) + .or_else([](specs::ParseError&& err) { throw std::move(err); }) + .value(); - if (auto depends = pkg["depends"].get_array(); !depends.error()) + auto signatures = [&] + { + auto maybe_sigs = repodata_doc["signatures"]; + if (!maybe_sigs.error() && verify_artifacts) { - for (auto elem : depends) - { - if (auto dep = elem.get_c_str(); !dep.error()) - { - package_info.dependencies.emplace_back(dep.value()); - } - } + return std::make_optional(maybe_sigs); } - - if (auto constrains = pkg["constrains"].get_array(); !constrains.error()) + else { - for (auto elem : constrains) - { - if (auto cons = elem.get_c_str(); !cons.error()) - { - package_info.constrains.emplace_back(cons.value()); - } - } + LOG_DEBUG << "No signatures available or requested. Downloading without verifying artifacts."; + return decltype(std::make_optional(maybe_sigs)){}; } + }(); - if (auto obj = pkg["track_features"]; !obj.error()) + // Process packages.conda first + if (auto pkgs = repodata_doc["packages.conda"]; !pkgs.error()) + { + if (auto packages_as_object = pkgs.get_object(); !packages_as_object.error()) { - if (auto track_features_arr = obj.get_array(); !track_features_arr.error()) + for (auto field : packages_as_object) { - for (auto elem : track_features_arr) + if (!field.error()) { - if (auto feat = elem.get_string(); !feat.error()) + const std::string key(field.unescaped_key().value()); + if (auto value = field.value(); !value.error()) { - package_info.track_features.emplace_back(feat.value()); + if (auto pkg_obj = value.get_object(); !pkg_obj.error()) + { + auto package_info = specs::PackageInfo::from_json( + filename.string(), + pkg_obj.value(), + parsed_url, + channel_id + ); + if (!package_info) + { + LOG_WARNING << package_info.error().what(); + } + alloc_solvable(package_info.value()); + } } } } - else if (auto track_features_str = obj.get_string(); !track_features_str.error()) - { - const auto lsplit_track_features = [](std::string_view features) - { - constexpr auto is_sep = [](char c) -> bool - { return (c == ',') || util::is_space(c); }; - auto [_, tail] = util::lstrip_if_parts(features, is_sep); - return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); - }; - - auto splits = lsplit_track_features(track_features_str.value()); - while (!splits[0].empty()) - { - package_info.track_features.emplace_back(splits[0]); - splits = lsplit_track_features(splits[1]); - } - } } - - database.alloc_solvable(package_info); - return true; - } - } - - Database::Database(specs::ChannelResolveParams channel_params) - : name_pool(Mapping<::resolvo::NameId, ::resolvo::String>()) - , m_channel_params(std::move(channel_params)) - { - } - - auto Database::channel_params() const -> const specs::ChannelResolveParams& - { - return m_channel_params; - } - - void Database::add_repo_from_repodata_json( - const fs::u8path& filename, - const std::string& repo_url, - const std::string& channel_id, - bool verify_artifacts - ) - { - auto parser = simdjson::dom::parser(); - const auto lock = LockFile(filename); - const auto repodata = parser.load(filename); - - // An override for missing package subdir is found at the top level - auto default_subdir = std::string(); - if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); !subdir.error()) - { - default_subdir = std::string(subdir.value_unsafe()); } - // Get `base_url` in case 'repodata_version': 2 - // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md - auto base_url = repo_url; - if (auto repodata_version = repodata["repodata_version"].get_int64(); - !repodata_version.error()) + // Then process packages + if (auto pkgs = repodata_doc["packages"]; !pkgs.error()) { - if (repodata_version.value_unsafe() == 2) + if (auto packages_as_object = pkgs.get_object(); !packages_as_object.error()) { - if (auto url = repodata.at_pointer("/info/base_url").get_string(); !url.error()) + for (auto field : packages_as_object) { - base_url = std::string(url.value_unsafe()); + if (!field.error()) + { + const std::string key(field.unescaped_key().value()); + if (auto value = field.value(); !value.error()) + { + if (auto pkg_obj = value.get_object(); !pkg_obj.error()) + { + auto package_info = specs::PackageInfo::from_json( + filename.string(), + pkg_obj.value(), + parsed_url, + channel_id + ); + if (!package_info) + { + LOG_WARNING << package_info.error().what(); + } + alloc_solvable(package_info.value()); + } + } + } } } } - - const auto parsed_url = specs::CondaURL::parse(base_url) - .or_else([](specs::ParseError&& err) { throw std::move(err); }) - .value(); - - auto signatures = std::optional(std::nullopt); - if (auto maybe_sigs = repodata["signatures"].get_object(); - !maybe_sigs.error() && verify_artifacts) - { - signatures = std::move(maybe_sigs).value(); - } - - auto added = util::flat_set(); - if (auto pkgs = repodata["packages.conda"].get_object(); !pkgs.error()) - { - for (auto [key, value] : pkgs.value()) - { - parse_packageinfo_json(key, value, parsed_url, channel_id, *this); - } - } - if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) - { - for (auto [key, value] : pkgs.value()) - { - parse_packageinfo_json(key, value, parsed_url, channel_id, *this); - } - } } void Database::add_repo_from_packages( @@ -532,149 +459,8 @@ namespace mamba::solver::resolvo return ::resolvo::VersionSetId{ 0 }; } - // NOTE: works around `openblas 0.2.18|0.2.18.*.` from - // `dlib==19.0=np110py27_blas_openblas_200` If contains "|", split on it and recurse - if (raw_match_spec_str.find("|") != std::string::npos) - { - std::vector match_specs; - std::string match_spec; - for (char c : raw_match_spec_str) - { - if (c == '|') - { - match_specs.push_back(match_spec); - match_spec.clear(); - } - else - { - match_spec += c; - } - } - match_specs.push_back(match_spec); - std::vector<::resolvo::VersionSetId> version_sets; - for (const std::string& ms : match_specs) - { - alloc_version_set(ms); - } - // Placeholder return value - return ::resolvo::VersionSetId{ 0 }; - } - - // NOTE: This works around some improperly encoded `constrains` in the test data, e.g.: - // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit - // >= 10.2" `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded - // `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" - // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: - // ">=4.5.2" - // Remove any with space after the binary operators - for (const std::string& op : { ">=", "<=", "==", ">", "<", "!=", "=", "==" }) - { - const std::string& bad_op = op + " "; - while (raw_match_spec_str.find(bad_op) != std::string::npos) - { - raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op - + raw_match_spec_str.substr( - raw_match_spec_str.find(bad_op) + bad_op.size() - ); - } - // If start with binary operator, prepend NONE - if (raw_match_spec_str.find(op) == 0) - { - raw_match_spec_str = "NONE " + raw_match_spec_str; - } - } - + // NOTE: works around ` const specs::MatchSpec match_spec = specs::MatchSpec::parse(raw_match_spec_str).value(); - // Add the version set to the version set pool - auto id = version_set_pool.alloc(match_spec); - - // Add name to the Name and String pools - const std::string name = match_spec.name().to_string(); - name_pool.alloc(::resolvo::String{ name }); - string_pool.alloc(::resolvo::String{ name }); - - // Add the MatchSpec's string representation to the Name and String pools - const std::string match_spec_str = match_spec.to_string(); - name_pool.alloc(::resolvo::String{ match_spec_str }); - string_pool.alloc(::resolvo::String{ match_spec_str }); - return id; - } - - ::resolvo::SolvableId Database::alloc_solvable(specs::PackageInfo package_info) - { - // Add the solvable to the solvable pool - auto id = solvable_pool.alloc(package_info); - - // Add name to the Name and String pools - const std::string name = package_info.name; - name_pool.alloc(::resolvo::String{ name }); - string_pool.alloc(::resolvo::String{ name }); - - // Add the long string representation of the package to the Name and String pools - const std::string long_str = package_info.long_str(); - name_pool.alloc(::resolvo::String{ long_str }); - string_pool.alloc(::resolvo::String{ long_str }); - - for (auto& dep : package_info.dependencies) - { - alloc_version_set(dep); - } - for (auto& constr : package_info.constrains) - { - alloc_version_set(constr); - } - - // Add the solvable to the name_to_solvable map - const ::resolvo::NameId name_id = name_pool.alloc(::resolvo::String{ package_info.name }); - name_to_solvable[name_id].push_back(id); - - return id; + return version_set_pool[match_spec]; } - - std::pair - Database::find_highest_version(::resolvo::VersionSetId version_set_id) - { - // If the version set has already been computed, return it. - if (version_set_to_max_version_and_track_features_numbers.find(version_set_id) - != version_set_to_max_version_and_track_features_numbers.end()) - { - return version_set_to_max_version_and_track_features_numbers[version_set_id]; - } - - const specs::MatchSpec match_spec = version_set_pool[version_set_id]; - - const std::string& name = match_spec.name().to_string(); - - auto name_id = name_pool.alloc(::resolvo::String{ name }); - - auto solvables = name_to_solvable[name_id]; - - auto filtered = filter_candidates(solvables, version_set_id, false); - - specs::Version max_version = specs::Version(); - size_t max_version_n_track_features = 0; - - for (auto& solvable_id : filtered) - { - const specs::PackageInfo& package_info = solvable_pool[solvable_id]; - const auto version = specs::Version::parse(package_info.version).value(); - if (version == max_version) - { - max_version_n_track_features = std::min( - max_version_n_track_features, - package_info.track_features.size() - ); - } - if (version > max_version) - { - max_version = version; - max_version_n_track_features = package_info.track_features.size(); - } - } - - auto val = std::make_pair(max_version, max_version_n_track_features); - version_set_to_max_version_and_track_features_numbers[version_set_id] = val; - return val; - } - -} // namespace mamba::solver::resolvo +} diff --git a/libmamba/src/specs/package_info.cpp b/libmamba/src/specs/package_info.cpp index f7cf9a6a61..79c55e4eef 100644 --- a/libmamba/src/specs/package_info.cpp +++ b/libmamba/src/specs/package_info.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "mamba/specs/archive.hpp" #include "mamba/specs/conda_url.hpp" @@ -565,4 +566,174 @@ namespace mamba::specs pkg.dependencies = j.value("depends", std::vector()); pkg.constrains = j.value("constrains", std::vector()); } + + auto PackageInfo::from_json( + const std::string_view& filename, + simdjson::ondemand::object& pkg, + const CondaURL& repo_url, + const std::string& channel_id + ) -> expected_parse_t + { + PackageInfo package_info; + + package_info.channel = channel_id; + package_info.filename = filename; + package_info.package_url = (repo_url / filename).str(CondaURL::Credentials::Show); + + if (auto fn = pkg["fn"]; !fn.error()) + { + package_info.name = fn.get_string().value_unsafe(); + } + else + { + // Fallback from key entry + package_info.name = filename; + } + + if (auto name = pkg["name"]; !name.error()) + { + package_info.name = name.get_string().value_unsafe(); + } + else + { + return make_unexpected_parse(fmt::format(R"(Found invalid name in "{}")", filename)); + } + + if (auto version = pkg["version"]; !version.error()) + { + package_info.version = version.get_string().value_unsafe(); + } + else + { + return make_unexpected_parse(fmt::format(R"(Found invalid version in "{}")", filename)); + } + + if (auto build_string = pkg["build"]; !build_string.error()) + { + package_info.build_string = build_string.get_string().value_unsafe(); + } + else + { + return make_unexpected_parse(fmt::format(R"(Found invalid build in "{}")", filename)); + } + + if (auto build_number = pkg["build_number"]; !build_number.error()) + { + package_info.build_number = build_number.get_uint64().value_unsafe(); + } + else + { + return make_unexpected_parse(fmt::format(R"(Found invalid build_number in "{}")", filename) + ); + } + + if (auto subdir = pkg["subdir"]; !subdir.error()) + { + package_info.platform = subdir.get_string().value_unsafe(); + } + + if (auto size = pkg["size"]; !size.error()) + { + package_info.size = size.get_uint64().value_unsafe(); + } + + if (auto md5 = pkg["md5"]; !md5.error()) + { + package_info.md5 = md5.get_string().value_unsafe(); + } + + if (auto sha256 = pkg["sha256"]; !sha256.error()) + { + package_info.sha256 = sha256.get_string().value_unsafe(); + } + + if (auto elem = pkg["noarch"]; !elem.error()) + { + if (auto noarch = elem.get_bool(); !noarch.error() && noarch.value_unsafe()) + { + package_info.noarch = NoArchType::Generic; + } + else if (elem.is_string()) + { + package_info.noarch = NoArchType::Generic; + } + } + + if (auto license = pkg["license"]; !license.error()) + { + package_info.license = license.get_string().value_unsafe(); + } + + // TODO conda timestamp are not Unix timestamp. + // Libsolv normalize them this way, we need to do the same here otherwise the current + // package may get arbitrary priority. + if (auto timestamp = pkg["timestamp"]; !timestamp.error()) + { + const auto time = timestamp.get_uint64().value_unsafe(); + constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; + package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; + } + + if (auto depends = pkg["depends"]; !depends.error()) + { + if (auto arr = depends.get_array(); !arr.error()) + { + for (auto elem : arr) + { + if (!elem.error() && elem.is_string()) + { + package_info.dependencies.emplace_back(elem.get_string().value_unsafe()); + } + } + } + } + + if (auto constrains = pkg["constrains"]; !constrains.error()) + { + if (auto arr = constrains.get_array(); !arr.error()) + { + for (auto elem : arr) + { + if (!elem.error() && elem.is_string()) + { + package_info.constrains.emplace_back(elem.get_string().value_unsafe()); + } + } + } + } + + if (auto track_features = pkg["track_features"]; !track_features.error()) + { + if (auto track_features_arr = track_features.get_array(); !track_features_arr.error()) + { + for (auto elem : track_features_arr) + { + if (auto feat = elem.get_string(); !feat.error()) + { + package_info.track_features.emplace_back(feat.value()); + } + } + } + else if (auto track_features_str = track_features.get_string(); + !track_features_str.error()) + { + const auto lsplit_track_features = [](std::string_view features) + { + constexpr auto is_sep = [](char c) -> bool + { return (c == ',') || util::is_space(c); }; + auto [_, tail] = util::lstrip_if_parts(features, is_sep); + return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); + }; + + auto splits = lsplit_track_features(track_features_str.value()); + while (!splits[0].empty()) + { + package_info.track_features.emplace_back(splits[0]); + splits = lsplit_track_features(splits[1]); + } + } + } + + return package_info; + } } diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index 9d141f4deb..a7f20dcabd 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -708,179 +708,22 @@ struct PackageDatabase : public DependencyProvider } }; -// TODO: factorise with the implementation from `set_solvable` in `mamba/solver/libsolv/helpers.cpp` bool parse_packageinfo_json( const std::string_view& filename, - const simdjson::dom::element& pkg, + const simdjson::ondemand::object& pkg, const CondaURL& repo_url, const std::string& channel_id, PackageDatabase& database ) { - PackageInfo package_info; - - package_info.channel = channel_id; - package_info.filename = filename; - package_info.package_url = (repo_url / filename).str(specs::CondaURL::Credentials::Show); - - if (auto fn = pkg["fn"].get_string(); !fn.error()) - { - package_info.name = fn.value_unsafe(); - } - else - { - // Fallback from key entry - package_info.name = filename; - } - - if (auto name = pkg["name"].get_string(); !name.error()) - { - package_info.name = name.value_unsafe(); - } - else - { - LOG_WARNING << R"(Found invalid name in ")" << filename << R"(")"; - return false; - } - - if (auto version = pkg["version"].get_string(); !version.error()) - { - package_info.version = version.value_unsafe(); - } - else - { - LOG_WARNING << R"(Found invalid version in ")" << filename << R"(")"; - return false; - } - - if (auto build_string = pkg["build"].get_string(); !build_string.error()) - { - package_info.build_string = build_string.value_unsafe(); - } - else - { - LOG_WARNING << R"(Found invalid build in ")" << filename << R"(")"; - return false; - } - - if (auto build_number = pkg["build_number"].get_uint64(); !build_number.error()) + auto maybe_package_info = PackageInfo::from_json(filename, pkg, repo_url, channel_id); + if (!maybe_package_info) { - package_info.build_number = build_number.value_unsafe(); - } - else - { - LOG_WARNING << R"(Found invalid build_number in ")" << filename << R"(")"; return false; } - if (auto subdir = pkg["subdir"].get_c_str(); !subdir.error()) - { - package_info.platform = subdir.value_unsafe(); - } - else - { - LOG_WARNING << R"(Found invalid subdir in ")" << filename << R"(")"; - } - - if (auto size = pkg["size"].get_uint64(); !size.error()) - { - package_info.size = size.value_unsafe(); - } - - if (auto md5 = pkg["md5"].get_c_str(); !md5.error()) - { - package_info.md5 = md5.value_unsafe(); - } - - if (auto sha256 = pkg["sha256"].get_c_str(); !sha256.error()) - { - package_info.sha256 = sha256.value_unsafe(); - } - - if (auto elem = pkg["noarch"]; !elem.error()) - { - // TODO: is the following right? - if (auto val = elem.get_bool(); !val.error() && val.value_unsafe()) - { - package_info.noarch = NoArchType::No; - } - else if (auto noarch = elem.get_c_str(); !noarch.error()) - { - package_info.noarch = NoArchType::No; - } - } - - if (auto license = pkg["license"].get_c_str(); !license.error()) - { - package_info.license = license.value_unsafe(); - } - - // TODO conda timestamp are not Unix timestamp. - // Libsolv normalize them this way, we need to do the same here otherwise the current - // package may get arbitrary priority. - if (auto timestamp = pkg["timestamp"].get_uint64(); !timestamp.error()) - { - const auto time = timestamp.value_unsafe(); - // TODO: reuse it from `mamba/solver/libsolv/helpers.cpp` - constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; - package_info.timestamp = (time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time; - } - - if (auto depends = pkg["depends"].get_array(); !depends.error()) - { - for (auto elem : depends) - { - if (auto dep = elem.get_c_str(); !dep.error()) - { - package_info.dependencies.emplace_back(dep.value_unsafe()); - } - } - } - - if (auto constrains = pkg["constrains"].get_array(); !constrains.error()) - { - for (auto elem : constrains) - { - if (auto cons = elem.get_c_str(); !cons.error()) - { - package_info.constrains.emplace_back(cons.value_unsafe()); - } - } - } - - if (auto track_features = pkg["track_features"]; !track_features.error()) - { - if (auto track_features_arr = track_features.get_array(); !track_features_arr.error()) - { - for (auto elem : track_features_arr) - { - if (auto feat = elem.get_string(); !feat.error()) - { - package_info.track_features.emplace_back(feat.value()); - } - } - } - else if (auto track_features_str = track_features.get_string(); !track_features_str.error()) - { - const auto lsplit_track_features = [](std::string_view features) - { - constexpr auto is_sep = [](char c) -> bool - { return (c == ',') || util::is_space(c); }; - auto [_, tail] = util::lstrip_if_parts(features, is_sep); - return util::lstrip_if_parts(tail, [&](char c) { return !is_sep(c); }); - }; - - auto splits = lsplit_track_features(track_features_str.value()); - while (!splits[0].empty()) - { - package_info.track_features.emplace_back(splits[0]); - splits = lsplit_track_features(splits[1]); - } - } - } - - database.alloc_solvable(package_info); + database.alloc_solvable(std::move(maybe_package_info).value()); return true; } @@ -893,27 +736,34 @@ parse_repodata_json( bool verify_artifacts ) { - auto parser = simdjson::dom::parser(); + auto parser = simdjson::ondemand::parser(); const auto lock = LockFile(filename); - const auto repodata = parser.load(filename); + const auto json_content = simdjson::padded_string::load(filename.string()); + const auto repodata = parser.iterate(json_content); // An override for missing package subdir is found at the top level auto default_subdir = std::string(); - if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); !subdir.error()) + if (auto info = repodata["info"]; !info.error()) { - default_subdir = std::string(subdir.value_unsafe()); + if (auto subdir = info["subdir"]; !subdir.error()) + { + default_subdir = std::string(subdir.get_string().value()); + } } // Get `base_url` in case 'repodata_version': 2 // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md auto base_url = repo_url; - if (auto repodata_version = repodata["repodata_version"].get_int64(); !repodata_version.error()) + if (auto repodata_version = repodata["repodata_version"]; !repodata_version.error()) { - if (repodata_version.value_unsafe() == 2) + if (repodata_version.get_int64().value() == 2) { - if (auto url = repodata.at_pointer("/info/base_url").get_string(); !url.error()) + if (auto info = repodata["info"]; !info.error()) { - base_url = std::string(url.value_unsafe()); + if (auto url = info["base_url"]; !url.error()) + { + base_url = std::string(url.get_string().value()); + } } } } @@ -922,29 +772,52 @@ parse_repodata_json( .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); - auto signatures = std::optional(std::nullopt); - if (auto maybe_sigs = repodata["signatures"].get_object(); !maybe_sigs.error() && verify_artifacts) + auto signatures = std::optional(std::nullopt); + if (auto maybe_sigs = repodata["signatures"]; !maybe_sigs.error() && verify_artifacts) { - signatures = std::move(maybe_sigs).value(); + if (auto obj = maybe_sigs.get_object(); !obj.error()) + { + signatures = std::move(obj).value(); + } } auto added = util::flat_set(); - if (auto pkgs = repodata["packages.conda"].get_object(); !pkgs.error()) + if (auto pkgs = repodata["packages.conda"]; !pkgs.error()) { std::cout << "CondaOrElseTarBz2 packages.conda" << std::endl; - for (auto [key, value] : pkgs.value()) + if (auto obj = pkgs.get_object(); !obj.error()) { - parse_packageinfo_json(key, value, parsed_url, channel_id, database); + for (auto field : obj) + { + if (!field.error()) + { + const std::string key(field.unescaped_key().value()); + if (auto value = field.value(); !value.error()) + { + parse_packageinfo_json(key, value, parsed_url, channel_id, database); + } + } + } } } - if (auto pkgs = repodata["packages"].get_object(); !pkgs.error()) + if (auto pkgs = repodata["packages"]; !pkgs.error()) { std::cout << "CondaOrElseTarBz2 packages" << std::endl; - for (auto [key, value] : pkgs.value()) + if (auto obj = pkgs.get_object(); !obj.error()) { - parse_packageinfo_json(key, value, parsed_url, channel_id, database); + for (auto field : obj) + { + if (!field.error()) + { + const std::string key(field.unescaped_key().value()); + if (auto value = field.value(); !value.error()) + { + parse_packageinfo_json(key, value, parsed_url, channel_id, database); + } + } + } } } } From 2748de11dc73605dd56daae53174d4c22cc1315e Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 21 May 2025 12:02:51 +0200 Subject: [PATCH 51/59] Complete implementation of `solver::resolvo::Database` Signed-off-by: Julien Jerphanion --- libmamba/src/solver/resolvo/database.cpp | 291 +++++++++++++++--- .../tests/src/solver/resolvo/test_solver.cpp | 90 +++--- 2 files changed, 304 insertions(+), 77 deletions(-) diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index 15ac5d8016..8daed45391 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -209,18 +209,179 @@ namespace mamba::solver::resolvo // TODO: Implement this } + /** + * Allocates a new requirement and return the id of the requirement. + */ + ::resolvo::VersionSetId Database::alloc_version_set(std::string_view raw_match_spec) + { + std::string raw_match_spec_str = std::string(raw_match_spec); + // Replace all " v" with simply " " to work around the `v` prefix in some version strings + // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in + // `inform2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` + while (raw_match_spec_str.find(" v") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(" v"), 2, " "); + } + + // Remove any presence of selector on python version in the match spec + // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in + // `infowillow-1.6.3-pyhd8ed1ab_0.conda` + for (const auto specifier : { "=py", "py", ">=py", "<=py", "!=py" }) + { + while (raw_match_spec_str.find(specifier) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(specifier)); + } + } + // Remove any white space between version + // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in + // `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` + while (raw_match_spec_str.find(", ") != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(", "), 2, ","); + } + + // TODO: skip allocation for now if "*.*" is in the match spec + if (raw_match_spec_str.find("*.*") != std::string::npos) + { + return ::resolvo::VersionSetId{ 0 }; + } + + // NOTE: works around `openblas 0.2.18|0.2.18.*.` from + // `dlib==19.0=np110py27_blas_openblas_200` If contains "|", split on it and recurse + if (raw_match_spec_str.find("|") != std::string::npos) + { + std::vector match_specs; + std::string match_spec; + for (char c : raw_match_spec_str) + { + if (c == '|') + { + match_specs.push_back(match_spec); + match_spec.clear(); + } + else + { + match_spec += c; + } + } + match_specs.push_back(match_spec); + for (const std::string& ms : match_specs) + { + alloc_version_set(ms); + } + // Placeholder return value + return ::resolvo::VersionSetId{ 0 }; + } + + // NOTE: This works around some improperly encoded `constrains` in the test data, e.g.: + // `openmpi-4.1.4-ha1ae619_102`'s improperly encoded `constrains`: "cudatoolkit + // >= 10.2" `pytorch-1.13.0-cpu_py310h02c325b_0.conda`'s improperly encoded + // `constrains`: "pytorch-cpu = 1.13.0", "pytorch-gpu = 99999999" + // `fipy-3.4.2.1-py310hff52083_3.tar.bz2`'s improperly encoded `constrains` or `dep`: + // ">=4.5.2" + // Remove any with space after the binary operators + for (const char* op : { ">=", "<=", "==", ">", "<", "!=", "=", "==" }) + { + const std::string bad_op = std::string(op) + " "; + while (raw_match_spec_str.find(bad_op) != std::string::npos) + { + raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(bad_op)) + op + + raw_match_spec_str.substr( + raw_match_spec_str.find(bad_op) + bad_op.size() + ); + } + // If start with binary operator, prepend NONE + if (raw_match_spec_str.find(op) == 0) + { + raw_match_spec_str = "NONE " + raw_match_spec_str; + } + } + + const specs::MatchSpec match_spec = specs::MatchSpec::parse(raw_match_spec_str).value(); + // Add the version set to the version set pool + auto id = version_set_pool.alloc(match_spec); + + // Add name to the Name and String pools + const std::string name = match_spec.name().to_string(); + name_pool.alloc(::resolvo::String{ name }); + string_pool.alloc(::resolvo::String{ name }); + + // Add the MatchSpec's string representation to the Name and String pools + const std::string match_spec_str = match_spec.to_string(); + name_pool.alloc(::resolvo::String{ match_spec_str }); + string_pool.alloc(::resolvo::String{ match_spec_str }); + return id; + } + + /** + * Allocates a new solvable and returns its id. + * + * - Adds the solvable to the solvable pool. + * - Adds the name to the Name and String pools. + * - Adds the long string representation of the package to the Name and String pools. + * - Allocates version sets for dependencies and constrains. + * - Adds the solvable to the name_to_solvable map. + */ + ::resolvo::SolvableId Database::alloc_solvable(specs::PackageInfo package_info) + { + // Add the solvable to the solvable pool + auto id = solvable_pool.alloc(package_info); + + // Add name to the Name and String pools + const std::string name = package_info.name; + name_pool.alloc(::resolvo::String{ name }); + string_pool.alloc(::resolvo::String{ name }); + + // Add the long string representation of the package to the Name and String pools + const std::string long_str = package_info.long_str(); + name_pool.alloc(::resolvo::String{ long_str }); + string_pool.alloc(::resolvo::String{ long_str }); + + for (auto& dep : package_info.dependencies) + { + alloc_version_set(dep); + } + for (auto& constr : package_info.constrains) + { + alloc_version_set(constr); + } + + // Add the solvable to the name_to_solvable map + const auto name_id = name_pool.alloc(::resolvo::String{ package_info.name }); + name_to_solvable[name_id].push_back(id); + + return id; + } + + /** + * Returns a user-friendly string representation of the specified solvable. + * + * When formatting the solvable, it should it include both the name of + * the package and any other identifying properties. + */ ::resolvo::String Database::display_solvable(::resolvo::SolvableId solvable) { const specs::PackageInfo& package_info = solvable_pool[solvable]; return ::resolvo::String{ package_info.long_str() }; } + /** + * Returns a user-friendly string representation of the name of the + * specified solvable. + */ ::resolvo::String Database::display_solvable_name(::resolvo::SolvableId solvable) { const specs::PackageInfo& package_info = solvable_pool[solvable]; return ::resolvo::String{ package_info.name }; } + /** + * Returns a string representation of multiple solvables merged together. + * + * When formatting the solvables, both the name of the packages and any + * other identifying properties should be included. + */ ::resolvo::String Database::display_merged_solvables(::resolvo::Slice<::resolvo::SolvableId> solvable) { @@ -232,34 +393,59 @@ namespace mamba::solver::resolvo return ::resolvo::String{ result }; } + /** + * Returns an object that can be used to display the given name in a + * user-friendly way. + */ ::resolvo::String Database::display_name(::resolvo::NameId name) { return name_pool[name]; } + /** + * Returns a user-friendly string representation of the specified version + * set. + * + * The name of the package should *not* be included in the display. Where + * appropriate, this information is added. + */ ::resolvo::String Database::display_version_set(::resolvo::VersionSetId version_set) { const specs::MatchSpec match_spec = version_set_pool[version_set]; return ::resolvo::String{ match_spec.to_string() }; } + /** + * Returns the string representation of the specified string. + */ ::resolvo::String Database::display_string(::resolvo::StringId string) { return string_pool[string]; } + /** + * Returns the name of the package that the specified version set is + * associated with. + */ ::resolvo::NameId Database::version_set_name(::resolvo::VersionSetId version_set_id) { const specs::MatchSpec match_spec = version_set_pool[version_set_id]; return name_pool[::resolvo::String{ match_spec.name().to_string() }]; } + /** + * Returns the name of the package for the given solvable. + */ ::resolvo::NameId Database::solvable_name(::resolvo::SolvableId solvable_id) { const specs::PackageInfo& package_info = solvable_pool[solvable_id]; return name_pool[::resolvo::String{ package_info.name }]; } + /** + * Obtains a list of solvables that should be considered when a package + * with the given name is requested. + */ ::resolvo::Candidates Database::get_candidates(::resolvo::NameId package) { ::resolvo::Candidates candidates{}; @@ -269,6 +455,63 @@ namespace mamba::solver::resolvo return candidates; } + /** + * Finds the highest version and the minimum number of track features for a given version set. + * + * - If the version set has already been computed, returns the cached value. + * - Filters candidates for the version set. + * - Iterates over filtered candidates to find the maximum version and the minimum number of + * track features. + * - Caches and returns the result. + */ + std::pair + Database::find_highest_version(::resolvo::VersionSetId version_set_id) + { + // If the version set has already been computed, return it. + if (version_set_to_max_version_and_track_features_numbers.find(version_set_id) + != version_set_to_max_version_and_track_features_numbers.end()) + { + return version_set_to_max_version_and_track_features_numbers[version_set_id]; + } + + const specs::MatchSpec match_spec = version_set_pool[version_set_id]; + const std::string& name = match_spec.name().to_string(); + auto name_id = name_pool.alloc(::resolvo::String{ name }); + auto solvables = name_to_solvable[name_id]; + auto filtered = filter_candidates(solvables, version_set_id, false); + + specs::Version max_version = specs::Version(); + size_t max_version_n_track_features = 0; + + for (auto& solvable_id : filtered) + { + const specs::PackageInfo& package_info = solvable_pool[solvable_id]; + const auto version = specs::Version::parse(package_info.version).value(); + if (version == max_version) + { + max_version_n_track_features = std::min( + max_version_n_track_features, + package_info.track_features.size() + ); + } + if (version > max_version) + { + max_version = version; + max_version_n_track_features = package_info.track_features.size(); + } + } + + auto val = std::make_pair(max_version, max_version_n_track_features); + version_set_to_max_version_and_track_features_numbers[version_set_id] = val; + return val; + } + + /** + * Sort the specified solvables based on which solvable to try first. The + * solver will iteratively try to select the highest version. If a + * conflict is found with the highest version the next version is + * tried. This continues until a solution is found. + */ void Database::sort_candidates(::resolvo::Slice<::resolvo::SolvableId> solvables) { std::sort( @@ -354,6 +597,11 @@ namespace mamba::solver::resolvo ); } + /** + * Given a set of solvables, return the solvables that match the given + * version set or if `inverse` is true, the solvables that do *not* match + * the version set. + */ ::resolvo::Vector<::resolvo::SolvableId> Database::filter_candidates( ::resolvo::Slice<::resolvo::SolvableId> candidates, ::resolvo::VersionSetId version_set_id, @@ -393,6 +641,9 @@ namespace mamba::solver::resolvo return filtered; } + /** + * Returns the dependencies for the specified solvable. + */ ::resolvo::Dependencies Database::get_dependencies(::resolvo::SolvableId solvable_id) { const specs::PackageInfo& package_info = solvable_pool[solvable_id]; @@ -423,44 +674,4 @@ namespace mamba::solver::resolvo return dependencies; } - - ::resolvo::VersionSetId Database::alloc_version_set(std::string_view raw_match_spec) - { - std::string raw_match_spec_str = std::string(raw_match_spec); - // Replace all " v" with simply " " to work around the `v` prefix in some version strings - // e.g. `mingw-w64-ucrt-x86_64-crt-git v12.0.0.r2.ggc561118da h707e725_0` in - // `inform2w64-sysroot_win-64-v12.0.0.r2.ggc561118da-h707e725_0.conda` while - // (raw_match_spec_str.find(" v") != std::string::npos) - { - raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(" v"), 2, " "); - } - - // Remove any presence of selector on python version in the match spec - // e.g. `pillow-heif >=0.10.0,<1.0.0 `pillow-heif >=0.10.0,<1.0.0` in - // `infowillow-1.6.3-pyhd8ed1ab_0.conda` - for (const auto specifier : { "=py", "py", ">=py", "<=py", "!=py" }) - { - while (raw_match_spec_str.find(specifier) != std::string::npos) - { - raw_match_spec_str = raw_match_spec_str.substr(0, raw_match_spec_str.find(specifier)); - } - } - // Remove any white space between version - // e.g. `kytea >=0.1.4, 0.2.0` -> `kytea >=0.1.4,0.2.0` in - // `infokonoha-4.6.3-pyhd8ed1ab_0.tar.bz2` - while (raw_match_spec_str.find(", ") != std::string::npos) - { - raw_match_spec_str = raw_match_spec_str.replace(raw_match_spec_str.find(", "), 2, ","); - } - - // TODO: skip allocation for now if "*.*" is in the match spec - if (raw_match_spec_str.find("*.*") != std::string::npos) - { - return ::resolvo::VersionSetId{ 0 }; - } - - // NOTE: works around ` - const specs::MatchSpec match_spec = specs::MatchSpec::parse(raw_match_spec_str).value(); - return version_set_pool[match_spec]; - } } diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index a7f20dcabd..ded89375f4 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -711,10 +711,10 @@ struct PackageDatabase : public DependencyProvider bool parse_packageinfo_json( const std::string_view& filename, - const simdjson::ondemand::object& pkg, - const CondaURL& repo_url, + simdjson::ondemand::object& pkg, + const specs::CondaURL& repo_url, const std::string& channel_id, - PackageDatabase& database + PackageDatabase& db ) { auto maybe_package_info = PackageInfo::from_json(filename, pkg, repo_url, channel_id); @@ -722,14 +722,13 @@ parse_packageinfo_json( { return false; } - - database.alloc_solvable(std::move(maybe_package_info).value()); + db.alloc_solvable(maybe_package_info.value()); return true; } void parse_repodata_json( - PackageDatabase& database, + PackageDatabase& db, const fs::u8path& filename, const std::string& repo_url, const std::string& channel_id, @@ -738,83 +737,100 @@ parse_repodata_json( { auto parser = simdjson::ondemand::parser(); const auto lock = LockFile(filename); + + // The json storage must be kept alive as long as we are reading the json data. const auto json_content = simdjson::padded_string::load(filename.string()); - const auto repodata = parser.iterate(json_content); - // An override for missing package subdir is found at the top level - auto default_subdir = std::string(); - if (auto info = repodata["info"]; !info.error()) - { - if (auto subdir = info["subdir"]; !subdir.error()) - { - default_subdir = std::string(subdir.get_string().value()); - } - } + // Note that with the "on-demand" parser, documents/values/objects act as iterators + // to go through the document. + auto repodata = parser.iterate(json_content); // Get `base_url` in case 'repodata_version': 2 // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md - auto base_url = repo_url; - if (auto repodata_version = repodata["repodata_version"]; !repodata_version.error()) + const auto base_url = [&] { - if (repodata_version.get_int64().value() == 2) + if (auto repodata_version = repodata["repodata_version"]; !repodata_version.error()) { if (auto info = repodata["info"]; !info.error()) { if (auto url = info["base_url"]; !url.error()) { - base_url = std::string(url.get_string().value()); + return std::string(url.get_string().value_unsafe()); } } } - } + + return repo_url; + }(); const auto parsed_url = specs::CondaURL::parse(base_url) .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); - auto signatures = std::optional(std::nullopt); - if (auto maybe_sigs = repodata["signatures"]; !maybe_sigs.error() && verify_artifacts) + auto signatures = [&] { - if (auto obj = maybe_sigs.get_object(); !obj.error()) + auto maybe_sigs = repodata["signatures"]; + if (!maybe_sigs.error() && verify_artifacts) { - signatures = std::move(obj).value(); + return std::make_optional(maybe_sigs); } - } + else + { + LOG_DEBUG << "No signatures available or requested. Downloading without verifying artifacts."; + return decltype(std::make_optional(maybe_sigs)){}; + } + }(); - auto added = util::flat_set(); + // Process packages.conda first if (auto pkgs = repodata["packages.conda"]; !pkgs.error()) { - std::cout << "CondaOrElseTarBz2 packages.conda" << std::endl; - - if (auto obj = pkgs.get_object(); !obj.error()) + if (auto packages_as_object = pkgs.get_object(); !packages_as_object.error()) { - for (auto field : obj) + for (auto field : packages_as_object) { if (!field.error()) { const std::string key(field.unescaped_key().value()); if (auto value = field.value(); !value.error()) { - parse_packageinfo_json(key, value, parsed_url, channel_id, database); + if (auto pkg_obj = value.get_object(); !pkg_obj.error()) + { + parse_packageinfo_json( + filename.string(), + pkg_obj.value(), + parsed_url, + channel_id, + db + ); + } } } } } } + + // Then process packages if (auto pkgs = repodata["packages"]; !pkgs.error()) { - std::cout << "CondaOrElseTarBz2 packages" << std::endl; - - if (auto obj = pkgs.get_object(); !obj.error()) + if (auto packages_as_object = pkgs.get_object(); !packages_as_object.error()) { - for (auto field : obj) + for (auto field : packages_as_object) { if (!field.error()) { const std::string key(field.unescaped_key().value()); if (auto value = field.value(); !value.error()) { - parse_packageinfo_json(key, value, parsed_url, channel_id, database); + if (auto pkg_obj = value.get_object(); !pkg_obj.error()) + { + parse_packageinfo_json( + filename.string(), + pkg_obj.value(), + parsed_url, + channel_id, + db + ); + } } } } From 71e83ac7482005d0e2ffd020fed2449e63ecf14b Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 22 May 2025 11:15:04 +0200 Subject: [PATCH 52/59] Unify naming of DatabaseVariant instances Signed-off-by: Julien Jerphanion --- libmamba/src/api/install.cpp | 34 +++++++++++++------------- libmamba/src/api/remove.cpp | 23 +++++++++--------- libmamba/src/api/update.cpp | 46 ++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index fee6bbac2b..85224fc4af 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -745,22 +745,22 @@ namespace mamba bool remove_prefix_on_failure ) { - solver::DatabaseVariant db = ctx.experimental_resolvo_solver - ? solver::DatabaseVariant( - std::in_place_type, - channel_context.params() - ) - : solver::DatabaseVariant( - std::in_place_type, - channel_context.params(), - solver::libsolv::Database::Settings{ - ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } - ); + solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); if (!ctx.experimental_resolvo_solver) { - add_spdlog_logger_to_database(std::get(db)); + add_spdlog_logger_to_database(std::get(db_variant)); } init_channels(ctx, channel_context); @@ -780,11 +780,11 @@ namespace mamba MultiPackageCache pkg_caches(ctx.pkgs_dirs, ctx.validation_params); - if (auto* libsolv_db = std::get_if(&db)) + if (auto* libsolv_db = std::get_if(&db_variant)) { load_installed_packages_in_database(ctx, *libsolv_db, prefix_data); } - else if (auto* resolvo_db = std::get_if(&db)) + else if (auto* resolvo_db = std::get_if(&db_variant)) { load_installed_packages_in_database(ctx, *resolvo_db, prefix_data); } @@ -792,7 +792,7 @@ namespace mamba std::vector others; // Note that the Transaction will gather the Solvables, // so they must have been ready in the database's pool before this line - auto transaction = create_transaction(db, pkg_caches, others); + auto transaction = create_transaction(db_variant, pkg_caches, others); std::vector lock_pkgs; diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index 925a6d48f3..1f925800f6 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -143,16 +143,16 @@ namespace mamba : solver::libsolv::MatchSpecParser::Libsolv, }, }; - solver::DatabaseVariant db = ctx.experimental_resolvo_solver - ? solver::DatabaseVariant( - std::in_place_type, - channel_context.params() - ) - : solver::DatabaseVariant(std::move(database)); + solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant(std::move(database)); if (!ctx.experimental_resolvo_solver) { - add_spdlog_logger_to_database(std::get(db)); + add_spdlog_logger_to_database(std::get(db_variant)); } load_installed_packages_in_database( @@ -162,7 +162,7 @@ namespace mamba std::reference_wrapper, std::reference_wrapper> { return std::ref(db); }, - db + db_variant ), prefix_data ); @@ -202,7 +202,6 @@ namespace mamba pkgs_to_remove.push_back(iter->second); } } - solver::DatabaseVariant db_variant = std::move(db); auto transaction = MTransaction(ctx, db_variant, pkgs_to_remove, {}, package_caches); return execute_transaction(transaction); } @@ -220,7 +219,7 @@ namespace mamba auto outcome = solver::libsolv::Solver() .solve( - std::get(db), + std::get(db_variant), request, ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba @@ -234,7 +233,7 @@ namespace mamba Console::instance().json_write( { { "success", false }, { "solver_problems", - unsolvable->problems(std::get(db)) } } + unsolvable->problems(std::get(db_variant)) } } ); } throw mamba_error( @@ -244,7 +243,7 @@ namespace mamba } Console::instance().json_write({ { "success", true } }); - solver::DatabaseVariant db_variant2 = std::move(db); + solver::DatabaseVariant db_variant2 = std::move(db_variant); auto transaction2 = MTransaction( ctx, db_variant2, diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 0efa36a249..06b805d956 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -157,27 +157,27 @@ namespace mamba populate_context_channels_from_specs(raw_update_specs, ctx); - solver::DatabaseVariant db = ctx.experimental_resolvo_solver - ? solver::DatabaseVariant( - std::in_place_type, - channel_context.params() - ) - : solver::DatabaseVariant(solver::libsolv::Database{ - channel_context.params(), - { - ctx.experimental_matchspec_parsing - ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - } }); + solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver + ? solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ) + : solver::DatabaseVariant(solver::libsolv::Database{ + channel_context.params(), + { + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv, + } }); if (!ctx.experimental_resolvo_solver) { - add_spdlog_logger_to_database(std::get(db)); + add_spdlog_logger_to_database(std::get(db_variant)); } MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); - auto exp_loaded = load_channels(ctx, channel_context, db, package_caches); + auto exp_loaded = load_channels(ctx, channel_context, db_variant, package_caches); if (!exp_loaded) { throw std::runtime_error(exp_loaded.error().what()); @@ -198,7 +198,7 @@ namespace mamba std::reference_wrapper, std::reference_wrapper> { return std::ref(db); }, - db + db_variant ), prefix_data ); @@ -224,7 +224,7 @@ namespace mamba using LibsolvOutcome = std::variant; auto outcome = ctx.experimental_resolvo_solver ? solver::resolvo::Solver() - .solve(std::get(db), request) + .solve(std::get(db_variant), request) .map( [](auto&& result) -> LibsolvOutcome { @@ -233,7 +233,7 @@ namespace mamba } ) : solver::libsolv::Solver().solve( - std::get(db), + std::get(db_variant), request, ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba @@ -248,7 +248,7 @@ namespace mamba if (auto* unsolvable = std::get_if(&result)) { unsolvable->explain_problems_to( - std::get(db), + std::get(db_variant), LOG_ERROR, { /* .unavailable= */ ctx.graphics_params.palette.failure, @@ -260,7 +260,7 @@ namespace mamba Console::instance().json_write(nlohmann::json{ { "success", false }, { "solver_problems", - unsolvable->problems(std::get(db)) } }); + unsolvable->problems(std::get(db_variant)) } }); } throw mamba_error( "Could not solve for environment specs", @@ -269,7 +269,13 @@ namespace mamba } Console::instance().json_write(nlohmann::json{ { "success", true } }); - auto transaction = MTransaction(ctx, db, request, std::get(result), package_caches); + auto transaction = MTransaction( + ctx, + db_variant, + request, + std::get(result), + package_caches + ); auto execute_transaction = [&](MTransaction& trans) From 0a2a79d9aad1a246fc1276f4823c8fe778ef8d9c Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 22 May 2025 11:27:00 +0200 Subject: [PATCH 53/59] Ignore packages signatures when using resolvo for now Signed-off-by: Julien Jerphanion --- libmamba/src/solver/resolvo/database.cpp | 27 ++++++++++--------- .../tests/src/solver/resolvo/test_solver.cpp | 27 ++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index 8daed45391..f50b0a426d 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -113,19 +113,20 @@ namespace mamba::solver::resolvo .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); - auto signatures = [&] - { - auto maybe_sigs = repodata_doc["signatures"]; - if (!maybe_sigs.error() && verify_artifacts) - { - return std::make_optional(maybe_sigs); - } - else - { - LOG_DEBUG << "No signatures available or requested. Downloading without verifying artifacts."; - return decltype(std::make_optional(maybe_sigs)){}; - } - }(); + // TODO: it does not seems resolvo can handle setting signatures on solvables for now + // auto signatures = [&] + // { + // auto maybe_sigs = repodata_doc["signatures"]; + // if (!maybe_sigs.error() && verify_artifacts) + // { + // return std::make_optional(maybe_sigs); + // } + // else + // { + // LOG_DEBUG << "No signatures available or requested. Downloading without verifying + // artifacts."; return decltype(std::make_optional(maybe_sigs)){}; + // } + // }(); // Process packages.conda first if (auto pkgs = repodata_doc["packages.conda"]; !pkgs.error()) diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index ded89375f4..e8c14d8ccb 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -767,19 +767,20 @@ parse_repodata_json( .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); - auto signatures = [&] - { - auto maybe_sigs = repodata["signatures"]; - if (!maybe_sigs.error() && verify_artifacts) - { - return std::make_optional(maybe_sigs); - } - else - { - LOG_DEBUG << "No signatures available or requested. Downloading without verifying artifacts."; - return decltype(std::make_optional(maybe_sigs)){}; - } - }(); + // TODO: it does not seems resolvo can handle setting signatures on solvables for now + // auto signatures = [&] + // { + // auto maybe_sigs = repodata["signatures"]; + // if (!maybe_sigs.error() && verify_artifacts) + // { + // return std::make_optional(maybe_sigs); + // } + // else + // { + // LOG_DEBUG << "No signatures available or requested. Downloading without verifying + // artifacts."; return decltype(std::make_optional(maybe_sigs)){}; + // } + // }(); // Process packages.conda first if (auto pkgs = repodata["packages.conda"]; !pkgs.error()) From 01f324fbf03300ae2196762f1e4e7b68f03cc11e Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 22 May 2025 12:18:00 +0200 Subject: [PATCH 54/59] Only create solver::libsolv::Database if needed Signed-off-by: Julien Jerphanion --- libmamba/src/api/install.cpp | 14 ++++++++------ libmamba/src/api/remove.cpp | 17 +++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 85224fc4af..bcbe5cb6c4 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -554,17 +554,19 @@ namespace mamba LOG_WARNING << "No 'channels' specified"; } - solver::libsolv::Database initial_db( - channel_context.params(), - { ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv } - ); solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver ? solver::DatabaseVariant( std::in_place_type, channel_context.params() ) - : solver::DatabaseVariant(std::move(initial_db)); + : solver::DatabaseVariant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv } + ); if (!ctx.experimental_resolvo_solver) { diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index 1f925800f6..9bb8f57748 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -136,19 +136,20 @@ namespace mamba } PrefixData& prefix_data = exp_prefix_data.value(); - solver::libsolv::Database database{ - channel_context.params(), - { - ctx.experimental_matchspec_parsing ? solver::libsolv::MatchSpecParser::Mamba - : solver::libsolv::MatchSpecParser::Libsolv, - }, - }; solver::DatabaseVariant db_variant = ctx.experimental_resolvo_solver ? solver::DatabaseVariant( std::in_place_type, channel_context.params() ) - : solver::DatabaseVariant(std::move(database)); + : solver::DatabaseVariant( + std::in_place_type, + channel_context.params(), + solver::libsolv::Database::Settings{ + ctx.experimental_matchspec_parsing + ? solver::libsolv::MatchSpecParser::Mamba + : solver::libsolv::MatchSpecParser::Libsolv, + } + ); if (!ctx.experimental_resolvo_solver) { From a790564684cc76ae34115ce73ab5ae6d093715c6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 22 May 2025 12:43:15 +0200 Subject: [PATCH 55/59] Revert change made to test_env_lockfile.cpp Signed-off-by: Julien Jerphanion --- libmamba/tests/src/core/test_env_lockfile.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libmamba/tests/src/core/test_env_lockfile.cpp b/libmamba/tests/src/core/test_env_lockfile.cpp index 1dd3660c39..b7ab316757 100644 --- a/libmamba/tests/src/core/test_env_lockfile.cpp +++ b/libmamba/tests/src/core/test_env_lockfile.cpp @@ -154,10 +154,9 @@ namespace mamba [&](std::vector categories, size_t num_conda, size_t num_pip) { std::vector other_specs; - solver::DatabaseVariant db_variant = std::move(db); auto transaction = create_explicit_transaction_from_lockfile( ctx, - db_variant, + db, lockfile_path, categories, pkg_cache, From 61a4d1af171351a35d8439349e9d7891789ca4cb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 28 May 2025 17:07:31 +0200 Subject: [PATCH 56/59] Post rebase unification Signed-off-by: Julien Jerphanion --- libmamba/src/api/install.cpp | 17 ++++++++++------- libmamba/src/api/remove.cpp | 7 +++---- libmamba/src/api/repoquery.cpp | 6 +++--- libmamba/tests/src/core/test_env_lockfile.cpp | 9 ++++++--- micromamba/src/update.cpp | 8 ++++---- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index bcbe5cb6c4..61b024ee3e 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -656,10 +656,6 @@ namespace mamba ); } } - auto& solution = std::get(result); - - Console::instance().json_write({ { "success", true } }); - auto transaction = MTransaction(ctx, db_variant, request, solution, package_caches); std::vector locks; @@ -668,21 +664,28 @@ namespace mamba locks.push_back(LockFile(c)); } + auto& solution = std::get(result); + + Console::instance().json_write({ { "success", true } }); + auto trans = MTransaction(ctx, db_variant, request, solution, package_caches); + if (ctx.output_params.json) { - transaction.log_json(); + trans.log_json(); } Console::stream(); - if (transaction.prompt(ctx, channel_context)) + if (trans.prompt(ctx, channel_context)) { if (create_env && !ctx.dry_run) { detail::create_target_directory(ctx, ctx.prefix_params.target_prefix); } - transaction.execute(ctx, channel_context, prefix_data); + detail::populate_state_file(ctx.prefix_params.target_prefix, env_vars, no_env); + + trans.execute(ctx, channel_context, prefix_data); // Print activation message only if the environment is freshly created if (create_env) diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index 9bb8f57748..5ce52f17fe 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -244,15 +244,14 @@ namespace mamba } Console::instance().json_write({ { "success", true } }); - solver::DatabaseVariant db_variant2 = std::move(db_variant); - auto transaction2 = MTransaction( + auto transaction = MTransaction( ctx, - db_variant2, + db_variant, request, std::get(outcome), package_caches ); - return execute_transaction(transaction2); + return execute_transaction(transaction); } } } diff --git a/libmamba/src/api/repoquery.cpp b/libmamba/src/api/repoquery.cpp index 427338d8ce..7f8fee4220 100644 --- a/libmamba/src/api/repoquery.cpp +++ b/libmamba/src/api/repoquery.cpp @@ -93,10 +93,10 @@ namespace mamba { Console::stream() << "Getting repodata from channels..." << std::endl; } - auto exp_load = load_channels(ctx, channel_context, database, package_caches); - if (!exp_load) + auto exp_loaded = load_channels(ctx, channel_context, database, package_caches); + if (!exp_loaded) { - throw std::runtime_error(exp_load.error().what()); + throw std::runtime_error(exp_loaded.error().what()); } } return database; diff --git a/libmamba/tests/src/core/test_env_lockfile.cpp b/libmamba/tests/src/core/test_env_lockfile.cpp index b7ab316757..0d7a1ae89a 100644 --- a/libmamba/tests/src/core/test_env_lockfile.cpp +++ b/libmamba/tests/src/core/test_env_lockfile.cpp @@ -144,8 +144,11 @@ namespace mamba const fs::u8path lockfile_path{ mambatests::test_data_dir / "env_lockfile/good_multiple_categories-lock.yaml" }; auto channel_context = ChannelContext::make_conda_compatible(mambatests::context()); - solver::libsolv::Database db{ channel_context.params() }; - add_spdlog_logger_to_database(db); + solver::DatabaseVariant db_variant = solver::DatabaseVariant( + std::in_place_type, + channel_context.params() + ); + add_spdlog_logger_to_database(std::get(db_variant)); mamba::MultiPackageCache pkg_cache({ "/tmp/" }, ctx.validation_params); ctx.platform = "linux-64"; @@ -156,7 +159,7 @@ namespace mamba std::vector other_specs; auto transaction = create_explicit_transaction_from_lockfile( ctx, - db, + db_variant, lockfile_path, categories, pkg_cache, diff --git a/micromamba/src/update.cpp b/micromamba/src/update.cpp index 0b4e3283ae..93c16a3789 100644 --- a/micromamba/src/update.cpp +++ b/micromamba/src/update.cpp @@ -128,10 +128,10 @@ update_self(Configuration& config, const std::optional& version) mamba::MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params); - auto exp_load = load_channels(ctx, channel_context, db_variant, package_caches); - if (!exp_load) + auto exp_loaded = load_channels(ctx, channel_context, db_variant, package_caches); + if (!exp_loaded) { - throw exp_load.error(); + throw exp_loaded.error(); } auto matchspec = specs::MatchSpec::parse( @@ -176,7 +176,7 @@ update_self(Configuration& config, const std::optional& version) ); ctx.download_only = true; - auto t = MTransaction(ctx, db_variant, { latest_micromamba.value() }, package_caches); + MTransaction t(ctx, db_variant, { latest_micromamba.value() }, package_caches); auto exp_prefix_data = PrefixData::create(ctx.prefix_params.root_prefix, channel_context); if (!exp_prefix_data) { From 0c3136c2ff221679951fafb1524996743b0e9345 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 28 May 2025 17:11:01 +0200 Subject: [PATCH 57/59] Rename `Mapping` to `bijective_map` Signed-off-by: Julien Jerphanion Co-authored-by: Johan Mabille --- .../include/mamba/solver/resolvo/database.hpp | 16 +++++++-------- libmamba/src/solver/resolvo/database.cpp | 2 +- .../tests/src/solver/resolvo/test_solver.cpp | 20 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index ffc00a310c..1c8a2277cd 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -65,11 +65,11 @@ namespace mamba::solver::resolvo { // Create a template Pool class that maps a key to a set of values template - struct Mapping + struct bijective_map { /** - * Adds the value to the Mapping and returns its associated id. If the - * value is already in the Mapping, returns the id associated with it. + * Adds the value to the bijective_map and returns its associated id. If the + * value is already in the bijective_map, returns the id associated with it. */ ID alloc(T value) { @@ -99,7 +99,7 @@ namespace mamba::solver::resolvo return value_to_id[value]; } - // Iterator for the Mapping + // Iterator for the bijective_map auto begin() { return id_to_value.begin(); @@ -249,10 +249,10 @@ namespace mamba::solver::resolvo find_highest_version(::resolvo::VersionSetId version_set_id); // Pools for mapping between resolvo IDs and mamba types - Mapping<::resolvo::NameId, ::resolvo::String> name_pool; - Mapping<::resolvo::StringId, ::resolvo::String> string_pool; - Mapping<::resolvo::VersionSetId, specs::MatchSpec> version_set_pool; - Mapping<::resolvo::SolvableId, specs::PackageInfo> solvable_pool; + bijective_map<::resolvo::NameId, ::resolvo::String> name_pool; + bijective_map<::resolvo::StringId, ::resolvo::String> string_pool; + bijective_map<::resolvo::VersionSetId, specs::MatchSpec> version_set_pool; + bijective_map<::resolvo::SolvableId, specs::PackageInfo> solvable_pool; bool has_package(const specs::MatchSpec& spec) { diff --git a/libmamba/src/solver/resolvo/database.cpp b/libmamba/src/solver/resolvo/database.cpp index f50b0a426d..0c2f0159db 100644 --- a/libmamba/src/solver/resolvo/database.cpp +++ b/libmamba/src/solver/resolvo/database.cpp @@ -18,7 +18,7 @@ namespace mamba::solver::resolvo { Database::Database(specs::ChannelResolveParams channel_params) - : name_pool(Mapping<::resolvo::NameId, ::resolvo::String>()) + : name_pool(bijective_map<::resolvo::NameId, ::resolvo::String>()) , m_channel_params(std::move(channel_params)) { } diff --git a/libmamba/tests/src/solver/resolvo/test_solver.cpp b/libmamba/tests/src/solver/resolvo/test_solver.cpp index e8c14d8ccb..933d5119f6 100644 --- a/libmamba/tests/src/solver/resolvo/test_solver.cpp +++ b/libmamba/tests/src/solver/resolvo/test_solver.cpp @@ -71,14 +71,14 @@ struct std::hash // Create a template Pool class that maps a key to a set of values template -struct Mapping +struct bijective_map { - Mapping() = default; - ~Mapping() = default; + bijective_map() = default; + ~bijective_map() = default; /** - * Adds the value to the Mapping and returns its associated id. If the - * value is already in the Mapping, returns the id associated with it. + * Adds the value to the bijective_map and returns its associated id. If the + * value is already in the bijective_map, returns the id associated with it. */ ID alloc(T value) { @@ -108,7 +108,7 @@ struct Mapping return value_to_id[value]; } - // Iterator for the Mapping + // Iterator for the bijective_map auto begin() { return id_to_value.begin(); @@ -210,14 +210,14 @@ struct PackageDatabase : public DependencyProvider { virtual ~PackageDatabase() = default; - ::Mapping name_pool; - ::Mapping string_pool; + ::bijective_map name_pool; + ::bijective_map string_pool; // MatchSpec are VersionSet in resolvo's semantics - ::Mapping version_set_pool; + ::bijective_map version_set_pool; // PackageInfo are Solvable in resolvo's semantics - ::Mapping solvable_pool; + ::bijective_map solvable_pool; // PackageName to Vector std::unordered_map> name_to_solvable; From d9c2e016f6dcee7b8f36386ea96560d5128fff7d Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 28 May 2025 17:33:46 +0200 Subject: [PATCH 58/59] Align APIs Signed-off-by: Julien Jerphanion Co-authored-by: Johan Mabille --- .../include/mamba/solver/resolvo/database.hpp | 58 +++---------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/libmamba/include/mamba/solver/resolvo/database.hpp b/libmamba/include/mamba/solver/resolvo/database.hpp index 1c8a2277cd..e2981a92e9 100644 --- a/libmamba/include/mamba/solver/resolvo/database.hpp +++ b/libmamba/include/mamba/solver/resolvo/database.hpp @@ -100,87 +100,47 @@ namespace mamba::solver::resolvo } // Iterator for the bijective_map - auto begin() + auto begin_values() const { return id_to_value.begin(); } - auto end() + auto end_values() const { return id_to_value.end(); } - auto begin() const - { - return id_to_value.begin(); - } - - auto end() const - { - return id_to_value.end(); - } - - auto cbegin() - { - return id_to_value.cbegin(); - } - - auto cend() - { - return id_to_value.cend(); - } - - auto cbegin() const + auto cbegin_values() const { return id_to_value.cbegin(); } - auto cend() const + auto cend_values() const { return id_to_value.cend(); } - auto find(T value) + auto find(T value) const { return value_to_id.find(value); } - auto begin_ids() + auto begin_keys() const { return value_to_id.begin(); } - auto end_ids() + auto end_keys() const { return value_to_id.end(); } - auto begin_ids() const - { - return value_to_id.begin(); - } - - auto end_ids() const - { - return value_to_id.end(); - } - - auto cbegin_ids() - { - return value_to_id.cbegin(); - } - - auto cend_ids() - { - return value_to_id.cend(); - } - - auto cbegin_ids() const + auto cbegin_keys() const { return value_to_id.cbegin(); } - auto cend_ids() const + auto cend_keys() const { return value_to_id.cend(); } From 107188f488642239f558e0d5581b7a8785828c70 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 11 Jun 2025 17:51:46 +0200 Subject: [PATCH 59/59] Post rebase adaptations Signed-off-by: Julien Jerphanion --- libmamba/src/core/transaction.cpp | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 40ac4de51c..daad69ff0f 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -87,21 +87,26 @@ namespace mamba find_python_version(const solver::Solution& solution, const solver::DatabaseVariant& database) -> std::pair { - auto old_python = installed_python(database); - auto new_python = std::optional(); + // We need to find the python version that will be there after this + // Transaction is finished in order to compile the noarch packages correctly, + + // We need to look into installed packages in case we are not installing a new python + // version but keeping the current one. + // Could also be written in term of PrefixData. + std::string installed_py_ver = {}; + if (auto python_version = installed_python(database)) + { + installed_py_ver = python_version.value(); + LOG_INFO << "Found python in installed packages " << installed_py_ver; + } - for_each_to_install( - solution.actions, - [&](const auto& pkg) - { - if (pkg.name == "python") - { - new_python = pkg.version; - } - } - ); + std::string new_py_ver = installed_py_ver; + if (auto py = solver::find_new_python_in_solution(solution)) + { + new_py_ver = py->get().version; + } - return { new_python.value_or(""), old_python.value_or("") }; + return { std::move(new_py_ver), std::move(installed_py_ver) }; } auto explicit_spec(const specs::PackageInfo& pkg) -> specs::MatchSpec