diff --git a/Cargo.lock b/Cargo.lock index 1a50fdf1c..07ddbc0e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4398,6 +4398,17 @@ dependencies = [ "yansi", ] +[[package]] +name = "priority-queue" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714c75db297bc88a63783ffc6ab9f830698a6705aa0201416931759ef4c8183d" +dependencies = [ + "autocfg", + "equivalent", + "indexmap 2.2.6", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -4426,6 +4437,19 @@ dependencies = [ "human_format", ] +[[package]] +name = "pubgrub" +version = "0.2.1" +source = "git+https://github.com/software-mansion-labs/pubgrub.git?branch=dev#cdae1729d7c47a6194a499d674ccdc96ce0838f1" +dependencies = [ + "indexmap 2.2.6", + "log", + "priority-queue", + "rustc-hash", + "thiserror", + "version-ranges", +] + [[package]] name = "pulldown-cmark" version = "0.12.2" @@ -4877,9 +4901,11 @@ dependencies = [ "libloading", "ntest", "once_cell", + "once_map", "pathdiff", "petgraph", "predicates", + "pubgrub", "ra_ap_toolchain", "redb", "reqwest", @@ -4890,6 +4916,7 @@ dependencies = [ "scarb-test-support", "scarb-ui", "semver", + "semver-pubgrub", "serde", "serde-untagged", "serde-value", @@ -4906,6 +4933,7 @@ dependencies = [ "test-for-each-example", "thiserror", "tokio", + "tokio-stream", "toml", "toml_edit 0.22.16", "tracing", @@ -5196,6 +5224,15 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-pubgrub" +version = "0.1.0" +source = "git+https://github.com/software-mansion-labs/semver-pubgrub.git#601f63233d74473929eb37ac223ba3d75849a140" +dependencies = [ + "pubgrub", + "semver", +] + [[package]] name = "serde" version = "1.0.215" @@ -5918,6 +5955,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -6270,6 +6318,14 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-ranges" +version = "0.1.0" +source = "git+https://github.com/software-mansion-labs/pubgrub.git?branch=dev#cdae1729d7c47a6194a499d674ccdc96ce0838f1" +dependencies = [ + "smallvec", +] + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index bc44061d3..5cfecfb0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,7 @@ pathdiff = { version = "0.2", features = ["camino"] } petgraph = "0.6" predicates = "3" proc-macro2 = "1" +pubgrub = { git = "https://github.com/pubgrub-rs/pubgrub.git", branch = "dev" } quote = "1" ra_ap_toolchain = "0.0.218" rayon = "1.10" @@ -106,6 +107,7 @@ redb = "2.2.0" reqwest = { version = "0.11", features = ["gzip", "brotli", "deflate", "json", "stream", "multipart"], default-features = false } salsa = { package = "rust-analyzer-salsa", version = "0.17.0-pre.6" } semver = { version = "1", features = ["serde"] } +semver-pubgrub = { git = "https://github.com/software-mansion-labs/semver-pubgrub.git" } serde = { version = "1", features = ["serde_derive"] } serde-untagged = "0.1" serde-value = "0.7" @@ -125,6 +127,7 @@ test-case = "3" thiserror = "1" time = "0.3" tokio = { version = "1", features = ["macros", "io-util", "process", "rt", "rt-multi-thread", "sync"] } +tokio-stream = "0.1" toml = "0.8" toml_edit = { version = "0.22", features = ["serde"] } tower-http = { version = "0.4", features = ["fs"] } @@ -142,6 +145,9 @@ xxhash-rust = { version = "0.8", features = ["xxh3"] } zip = { version = "0.6", default-features = false, features = ["deflate"] } zstd = "0.13" +[patch.'https://github.com/pubgrub-rs/pubgrub.git'] +pubgrub = { git = 'https://github.com/software-mansion-labs/pubgrub.git', branch = 'dev' } + [profile.release] lto = true diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index d1d093ab7..9f2b18048 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -55,8 +55,10 @@ indoc.workspace = true itertools.workspace = true libloading.workspace = true once_cell.workspace = true +once_map = { path = "../utils/once-map" } pathdiff.workspace = true petgraph.workspace = true +pubgrub.workspace = true scarb-proc-macro-server-types = { path = "../utils/scarb-proc-macro-server-types" } ra_ap_toolchain.workspace = true redb.workspace = true @@ -65,6 +67,7 @@ scarb-build-metadata = { path = "../utils/scarb-build-metadata" } scarb-metadata = { path = "../scarb-metadata", default-features = false, features = ["builder"] } scarb-stable-hash = { path = "../utils/scarb-stable-hash" } scarb-ui = { path = "../utils/scarb-ui" } +semver-pubgrub.workspace = true semver.workspace = true serde-untagged.workspace = true serde-value.workspace = true @@ -77,6 +80,7 @@ smol_str.workspace = true tar.workspace = true thiserror.workspace = true tokio.workspace = true +tokio-stream.workspace = true toml.workspace = true toml_edit.workspace = true tracing-subscriber.workspace = true diff --git a/scarb/src/core/lockfile.rs b/scarb/src/core/lockfile.rs index 4aa47d9e2..44292b7d7 100644 --- a/scarb/src/core/lockfile.rs +++ b/scarb/src/core/lockfile.rs @@ -21,7 +21,7 @@ pub enum LockVersion { V1 = 1, } -#[derive(Debug, Eq, PartialEq, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct Lockfile { pub version: LockVersion, diff --git a/scarb/src/core/registry/mod.rs b/scarb/src/core/registry/mod.rs index c8d47c05e..0d5820adc 100644 --- a/scarb/src/core/registry/mod.rs +++ b/scarb/src/core/registry/mod.rs @@ -111,7 +111,7 @@ pub(crate) mod mock { let summary = Summary::builder() .package_id(package_id) .dependencies(dependencies) - .no_core(package_id.is_core()) + .no_core(package_id.name == PackageName::CORE) .build(); let manifest = Box::new( diff --git a/scarb/src/resolver/algorithm/in_memory_index.rs b/scarb/src/resolver/algorithm/in_memory_index.rs new file mode 100644 index 000000000..31186f764 --- /dev/null +++ b/scarb/src/resolver/algorithm/in_memory_index.rs @@ -0,0 +1,28 @@ +use crate::core::{ManifestDependency, Summary}; +use once_map::OnceMap; +use std::sync::Arc; + +/// In-memory index of package metadata. +#[derive(Default, Clone)] +pub struct InMemoryIndex(Arc); + +#[derive(Default)] +struct SharedInMemoryIndex { + /// A map from package name to the metadata for that package and the index where the metadata + /// came from. + packages: FxOnceMap>, +} + +pub(crate) type FxOnceMap = OnceMap; + +impl InMemoryIndex { + /// Returns a reference to the package metadata. + pub fn packages(&self) -> &FxOnceMap> { + &self.0.packages + } +} + +#[derive(Debug)] +pub enum VersionsResponse { + Found(Vec), +} diff --git a/scarb/src/resolver/algorithm/mod.rs b/scarb/src/resolver/algorithm/mod.rs index 0156e3ba2..0c659c83f 100644 --- a/scarb/src/resolver/algorithm/mod.rs +++ b/scarb/src/resolver/algorithm/mod.rs @@ -1,12 +1,153 @@ use crate::core::lockfile::Lockfile; use crate::core::registry::Registry; -use crate::core::{Resolve, Summary}; +use crate::core::{PackageId, Resolve, Summary}; +use crate::resolver::algorithm::provider::{ + rewrite_locked_dependency, DependencyProviderError, PubGrubDependencyProvider, PubGrubPackage, +}; +use crate::resolver::algorithm::solution::{build_resolve, validate_solution}; +use crate::resolver::algorithm::state::{Request, ResolverState}; +use anyhow::bail; +use futures::{FutureExt, TryFutureExt}; +use itertools::Itertools; +use pubgrub::PubGrubError; +use pubgrub::{DefaultStringReporter, Reporter}; +use pubgrub::{Incompatibility, State}; +use std::collections::HashSet; +use std::sync::Arc; +use std::thread; +use tokio::sync::{mpsc, oneshot}; -#[tracing::instrument(level = "trace", skip_all)] -pub async fn resolve( - _summaries: &[Summary], - _registry: &dyn Registry, - _lockfile: Lockfile, +mod in_memory_index; +mod provider; +mod solution; +mod state; + +pub async fn resolve<'c>( + summaries: &[Summary], + registry: &dyn Registry, + lockfile: Lockfile, ) -> anyhow::Result { - todo!("implement") + let state = Arc::new(ResolverState::default()); + + let (request_sink, request_stream): (mpsc::Sender, mpsc::Receiver) = + mpsc::channel(300); + + let requests_fut = state + .clone() + .fetch(registry, request_stream) + .map_err(|err| anyhow::format_err!(err)) + .fuse(); + + for summary in summaries { + for dep in summary.full_dependencies() { + let locked_package_id = lockfile.packages_matching(dep.clone()); + let dep = if let Some(locked_package_id) = locked_package_id { + rewrite_locked_dependency(dep.clone(), locked_package_id?) + } else { + dep.clone() + }; + if state.index.packages().register(dep.clone()) { + request_sink.send(Request::Package(dep.clone())).await?; + } + } + } + + let main_package_ids: HashSet = + HashSet::from_iter(summaries.iter().map(|sum| sum.package_id)); + + let (tx, rx) = oneshot::channel(); + + let cloned_lockfile = lockfile.clone(); + thread::Builder::new() + .name("scarb-resolver".into()) + .spawn(move || { + let result = || { + let provider = PubGrubDependencyProvider::new( + main_package_ids, + state, + request_sink, + cloned_lockfile, + ); + + // Init state + let main_package_ids = provider + .main_package_ids() + .clone() + .into_iter() + .collect_vec(); + + let Some((first, rest)) = main_package_ids.split_first() else { + bail!("empty summaries"); + }; + let package: PubGrubPackage = (*first).into(); + let version = first.version.clone(); + let mut state = State::init(package.clone(), version); + state + .unit_propagation(package.clone()) + .map_err(|err| anyhow::format_err!("unit propagation failed: {:?}", err))?; + for package_id in rest { + let package: PubGrubPackage = (*package_id).into(); + let version = package_id.version.clone(); + state.add_incompatibility(Incompatibility::not_root( + package.clone(), + version.clone(), + )); + state + .unit_propagation(package) + .map_err(|err| anyhow::format_err!("unit propagation failed: {:?}", err))? + } + + // Resolve requirements + let solution = + pubgrub::resolve_state(&provider, &mut state, package).map_err(format_error)?; + + validate_solution(&solution)?; + build_resolve(&provider, solution) + }; + let result = result(); + tx.send(result).unwrap(); + })?; + + let resolve_fut = async move { + rx.await + .map_err(|_| DependencyProviderError::ChannelClosed.into()) + .and_then(|result| result) + }; + + let (_, resolve) = tokio::try_join!(requests_fut, resolve_fut)?; + resolve.check_checksums(&lockfile)?; + Ok(resolve) +} + +fn format_error(err: PubGrubError) -> anyhow::Error { + match err { + PubGrubError::NoSolution(derivation_tree) => { + anyhow::format_err!( + "version solving failed:\n{}\n", + DefaultStringReporter::report(&derivation_tree) + ) + } + PubGrubError::ErrorChoosingPackageVersion(DependencyProviderError::PackageNotFound { + name, + version, + }) => { + anyhow::format_err!("cannot find package `{name} {version}`") + } + PubGrubError::ErrorChoosingPackageVersion(DependencyProviderError::PackageQueryFailed( + err, + )) => anyhow::format_err!("{}", err).context("dependency query failed"), + PubGrubError::ErrorRetrievingDependencies { + package, + version, + source, + } => anyhow::Error::from(source) + .context(format!("cannot get dependencies of `{package}@{version}`")), + PubGrubError::ErrorInShouldCancel(err) => { + anyhow::format_err!("{}", err).context("should cancel failed") + } + PubGrubError::Failure(msg) => anyhow::format_err!("{}", msg).context("resolver failure"), + PubGrubError::ErrorChoosingPackageVersion(DependencyProviderError::ChannelClosed) => { + anyhow::format_err!("channel closed") + } + } } diff --git a/scarb/src/resolver/algorithm/provider.rs b/scarb/src/resolver/algorithm/provider.rs new file mode 100644 index 000000000..f229aaec3 --- /dev/null +++ b/scarb/src/resolver/algorithm/provider.rs @@ -0,0 +1,418 @@ +use crate::core::lockfile::Lockfile; +use crate::core::{ + DependencyFilter, DependencyVersionReq, ManifestDependency, PackageId, PackageName, SourceId, + Summary, +}; +use crate::resolver::algorithm::in_memory_index::VersionsResponse; +use crate::resolver::algorithm::{Request, ResolverState}; +use itertools::Itertools; +use pubgrub::{Dependencies, DependencyProvider, Range}; +use pubgrub::{Ranges, VersionSet}; +use semver::{Version, VersionReq}; +use semver_pubgrub::SemverPubgrub; +use std::cmp::Reverse; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::sync::{Arc, RwLock}; +use thiserror::Error; +use tokio::sync::mpsc; + +#[derive(Eq, PartialEq, Clone, Debug)] +pub struct CustomIncompatibility(String); + +impl Display for CustomIncompatibility { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Package identifier for PubGrub algorithm. +/// This identifier is stripped from version, which is represented with [`semver::Version`] instead. +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct PubGrubPackage { + pub name: PackageName, + pub source_id: SourceId, +} + +impl PubGrubPackage { + fn to_dependency(&self, range: SemverPubgrub) -> ManifestDependency { + ManifestDependency::builder() + .name(self.name.clone()) + .source_id(self.source_id) + .version_req(range.into()) + .build() + } +} + +impl From for DependencyVersionReq { + /// This conversion will always return a [`DependencyVersionReq`] that includes the provided + /// range. It's not guaranteed that the range will be exactly the same as the original one. + /// It will never be more restrictive than the original range though. + fn from(range: SemverPubgrub) -> Self { + let Some((start, end)) = range.bounding_range() else { + return DependencyVersionReq::Any; + }; + let bounds = (start.map(|b| b.clone()), end.map(|b| b.clone())); + let range: Ranges = Range::from_range_bounds(bounds); + VersionReq::parse(&range.to_string()) + .map(|req| { + if req.comparators.is_empty() { + return DependencyVersionReq::Any; + } + DependencyVersionReq::Req(req) + }) + .unwrap_or(DependencyVersionReq::Any) + } +} + +impl From<&ManifestDependency> for PubGrubPackage { + fn from(dependency: &ManifestDependency) -> Self { + Self { + name: dependency.name.clone(), + source_id: dependency.source_id, + } + } +} + +impl From for PubGrubPackage { + fn from(package_id: PackageId) -> Self { + Self { + name: package_id.name.clone(), + source_id: package_id.source_id, + } + } +} + +impl From<&Summary> for PubGrubPackage { + fn from(summary: &Summary) -> Self { + summary.package_id.into() + } +} + +impl Display for PubGrubPackage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[allow(dead_code)] +pub enum PubGrubPriority { + /// The package has no specific priority. + /// + /// As such, its priority is based on the order in which the packages were added (FIFO), such + /// that the first package we visit is prioritized over subsequent packages. + Unspecified(Reverse), + + /// The version range is constrained to a single version (e.g., with the `==` operator). + Singleton(Reverse), + + /// The package was specified via a direct URL. + DirectUrl(Reverse), + + /// The package is the root package. + Root, +} + +pub struct PubGrubDependencyProvider { + priority: RwLock>, + packages: RwLock>, + main_package_ids: HashSet, + lockfile: Lockfile, + state: Arc, + request_sink: mpsc::Sender, +} + +impl PubGrubDependencyProvider { + pub fn new( + main_package_ids: HashSet, + state: Arc, + request_sink: mpsc::Sender, + lockfile: Lockfile, + ) -> Self { + Self { + main_package_ids, + priority: RwLock::new(HashMap::new()), + packages: RwLock::new(HashMap::new()), + state, + lockfile, + request_sink, + } + } + + pub fn main_package_ids(&self) -> &HashSet { + &self.main_package_ids + } + + pub fn fetch_summary_and_request_dependencies( + &self, + package_id: PackageId, + ) -> Result { + let summary = self.fetch_summary(package_id)?; + self.request_dependencies(&summary)?; + Ok(summary) + } + + pub fn fetch_summary(&self, package_id: PackageId) -> Result { + let summary = self.packages.read().unwrap().get(&package_id).cloned(); + let summary = summary.map(Ok).unwrap_or_else(|| { + let dependency = ManifestDependency::builder() + .name(package_id.name.clone()) + .source_id(package_id.source_id) + .version_req(DependencyVersionReq::exact(&package_id.version)) + .build(); + let summary = self + .wait_for_summaries(dependency.clone())? + .into_iter() + .find_or_first(|summary| summary.package_id == package_id); + if let Some(summary) = summary.as_ref() { + let mut write_lock = self.packages.write().unwrap(); + write_lock.insert(summary.package_id, summary.clone()); + write_lock.insert(package_id, summary.clone()); + } + summary.ok_or_else(|| DependencyProviderError::PackageNotFound { + name: dependency.name.clone().to_string(), + version: dependency.version_req.clone(), + }) + })?; + Ok(summary) + } + + fn request_dependencies(&self, summary: &Summary) -> Result<(), DependencyProviderError> { + for dep in summary.dependencies.iter() { + let locked_package_id = self.lockfile.packages_matching(dep.clone()); + let dep = if let Some(locked_package_id) = locked_package_id { + rewrite_locked_dependency(dep.clone(), locked_package_id?) + } else { + dep.clone() + }; + + if self.state.index.packages().register(dep.clone()) { + self.request_sink + .blocking_send(Request::Package(dep.clone())) + .unwrap(); + } + + let dep = rewrite_path_dependency_source_id(summary.package_id, &dep)?; + if self.state.index.packages().register(dep.clone()) { + self.request_sink + .blocking_send(Request::Package(dep)) + .unwrap(); + } + } + Ok(()) + } + + fn wait_for_summaries( + &self, + dependency: ManifestDependency, + ) -> Result, DependencyProviderError> { + let summaries = self + .state + .index + .packages() + .wait_blocking(&dependency) + .unwrap(); + let VersionsResponse::Found(summaries) = summaries.as_ref(); + + { + let mut write_lock = self.packages.write().unwrap(); + for summary in summaries.iter() { + write_lock.insert(summary.package_id, summary.clone()); + } + } + + // Sort from highest to lowest. + let summaries = summaries + .iter() + .sorted_by_key(|sum| sum.package_id.version.clone()) + .rev() + .cloned() + .collect_vec(); + + Ok(summaries) + } +} + +impl DependencyProvider for PubGrubDependencyProvider { + type P = PubGrubPackage; + type V = Version; + type VS = SemverPubgrub; + type M = CustomIncompatibility; + + fn prioritize(&self, package: &Self::P, range: &Self::VS) -> Self::Priority { + let dependency: ManifestDependency = package.to_dependency(range.clone()); + if self.state.index.packages().register(dependency.clone()) { + self.request_sink + .blocking_send(Request::Package(dependency.clone())) + .unwrap(); + } + + // Prioritize by ordering from root. + let priority = self.priority.read().unwrap().get(package).copied(); + if let Some(priority) = priority { + return Some(PubGrubPriority::Unspecified(Reverse(priority))); + } + None + } + + type Priority = Option; + type Err = DependencyProviderError; + + fn choose_version( + &self, + package: &Self::P, + range: &Self::VS, + ) -> Result, Self::Err> { + // Query available versions. + let dependency: ManifestDependency = package.to_dependency(range.clone()); + let summaries = self.wait_for_summaries(dependency)?; + + // Choose version. + let summary = summaries + .into_iter() + .find(|summary| range.contains(&summary.package_id.version)); + + // Store retrieved summary for selected version. + if let Some(summary) = summary.as_ref() { + self.packages + .write() + .unwrap() + .insert(summary.package_id, summary.clone()); + } + + Ok(summary.map(|summary| summary.package_id.version.clone())) + } + + fn get_dependencies( + &self, + package: &Self::P, + version: &Self::V, + ) -> Result, Self::Err> { + // Query summary. + let package_id = PackageId::new(package.name.clone(), version.clone(), package.source_id); + let summary = self.fetch_summary_and_request_dependencies(package_id)?; + + // Set priority for dependencies. + let self_priority = self + .priority + .read() + .unwrap() + .get(&PubGrubPackage { + name: package_id.name.clone(), + source_id: package_id.source_id, + }) + .copied(); + if let Some(priority) = self_priority { + let mut write_lock = self.priority.write().unwrap(); + for dependency in summary.full_dependencies() { + let package: PubGrubPackage = dependency.into(); + write_lock.insert(package, priority + 1); + } + } + + // Convert dependencies to constraints. + let dep_filter = + DependencyFilter::propagation(self.main_package_ids.contains(&summary.package_id)); + let deps = summary + .filtered_full_dependencies(dep_filter) + .cloned() + .map(|dependency| { + let original_dep = dependency.clone(); + let dependency = + rewrite_path_dependency_source_id(summary.package_id, &dependency)?; + let locked_package_id = self.lockfile.packages_matching(dependency.clone()); + let dependency = if let Some(locked_package_id) = locked_package_id { + rewrite_locked_dependency(dependency.clone(), locked_package_id?) + } else { + dependency + }; + + let dep_name = dependency.name.clone().to_string(); + let summaries = self.wait_for_summaries(dependency.clone())?; + let summaries = if summaries.is_empty() { + self.wait_for_summaries(original_dep.clone())? + } else { + summaries + }; + summaries + .into_iter() + .find(|summary| dependency.version_req.matches(&summary.package_id.version)) + .map(|summary| (summary.package_id, dependency.version_req.clone())) + .ok_or_else(|| DependencyProviderError::PackageNotFound { + name: dep_name, + version: dependency.version_req.clone(), + }) + }) + .collect::, DependencyProviderError>>()?; + let constraints = deps + .into_iter() + .map(|(package_id, req)| (package_id.into(), req.into())) + .collect(); + + Ok(Dependencies::Available(constraints)) + } +} + +impl From for SemverPubgrub { + fn from(req: DependencyVersionReq) -> Self { + match req { + DependencyVersionReq::Req(req) => SemverPubgrub::from(&req), + DependencyVersionReq::Any => SemverPubgrub::empty().complement(), + DependencyVersionReq::Locked { exact, .. } => { + DependencyVersionReq::exact(&exact).into() + } + } + } +} + +pub fn rewrite_locked_dependency( + dependency: ManifestDependency, + locked_package_id: PackageId, +) -> ManifestDependency { + ManifestDependency::builder() + .kind(dependency.kind.clone()) + .name(dependency.name.clone()) + .source_id(locked_package_id.source_id) + .version_req(DependencyVersionReq::Locked { + exact: locked_package_id.version.clone(), + req: dependency.version_req.clone().into(), + }) + .build() +} + +pub fn rewrite_path_dependency_source_id( + package_id: PackageId, + dependency: &ManifestDependency, +) -> Result { + // Rewrite path dependencies for git sources. + if package_id.source_id.is_git() && dependency.source_id.is_path() { + let rewritten_dep = ManifestDependency::builder() + .kind(dependency.kind.clone()) + .name(dependency.name.clone()) + .source_id(package_id.source_id) + .version_req(dependency.version_req.clone()) + .build(); + + return Ok(rewritten_dep); + }; + Ok(dependency.clone()) +} + +/// Error thrown while trying to execute `scarb` command. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum DependencyProviderError { + /// Package not found. + #[error("cannot find package `{name} {version}`")] + PackageNotFound { + name: String, + version: DependencyVersionReq, + }, + /// Package query failed. + #[error("{0}")] + PackageQueryFailed(#[from] anyhow::Error), + /// Channel closed. + #[error("channel closed")] + ChannelClosed, +} diff --git a/scarb/src/resolver/algorithm/solution.rs b/scarb/src/resolver/algorithm/solution.rs new file mode 100644 index 000000000..643b25bf5 --- /dev/null +++ b/scarb/src/resolver/algorithm/solution.rs @@ -0,0 +1,79 @@ +use crate::core::resolver::DependencyEdge; +use crate::core::{ + DepKind, DependencyFilter, PackageId, PackageName, Resolve, Summary, TargetKind, +}; +use crate::resolver::algorithm::provider::{PubGrubDependencyProvider, PubGrubPackage}; +use anyhow::bail; +use indoc::indoc; +use petgraph::prelude::DiGraphMap; +use pubgrub::SelectedDependencies; +use std::collections::HashMap; + +pub fn build_resolve( + provider: &PubGrubDependencyProvider, + solution: SelectedDependencies, +) -> anyhow::Result { + let summaries: HashMap = solution + .into_iter() + .map(|(package, version)| { + let pid = PackageId::new(package.name.clone(), version.clone(), package.source_id); + let sum = provider + .fetch_summary(pid) + .map_err(|err| anyhow::format_err!("failed to get summary: {:?}", err))?; + Ok((sum.package_id, sum)) + }) + .collect::>>()?; + + let mut graph: DiGraphMap = Default::default(); + + for pid in summaries.keys() { + graph.add_node(*pid); + } + + for summary in summaries.values() { + let dep_filter = DependencyFilter::propagation( + provider.main_package_ids().contains(&summary.package_id), + ); + for dep in summary.filtered_full_dependencies(dep_filter) { + let dep_target_kind: Option = match dep.kind.clone() { + DepKind::Normal => None, + DepKind::Target(target_kind) => Some(target_kind), + }; + let Some(dep) = summaries.keys().find(|pid| pid.name == dep.name).copied() else { + continue; + }; + let weight = graph + .edge_weight(summary.package_id, dep) + .cloned() + .unwrap_or_default(); + let weight = weight.extend(dep_target_kind); + graph.add_edge(summary.package_id, dep, weight); + } + } + + Ok(Resolve { graph, summaries }) +} + +pub fn validate_solution( + solution: &SelectedDependencies, +) -> anyhow::Result<()> { + // Same package, different sources. + let mut seen: HashMap = Default::default(); + for pkg in solution.keys() { + if let Some(existing) = seen.get(&pkg.name) { + bail!( + indoc! {" + found dependencies on the same package `{}` coming from incompatible \ + sources: + source 1: {} + source 2: {} + "}, + pkg.name, + existing.source_id, + pkg.source_id + ); + } + seen.insert(pkg.name.clone(), pkg.clone()); + } + Ok(()) +} diff --git a/scarb/src/resolver/algorithm/state.rs b/scarb/src/resolver/algorithm/state.rs new file mode 100644 index 000000000..3db25ba92 --- /dev/null +++ b/scarb/src/resolver/algorithm/state.rs @@ -0,0 +1,63 @@ +use crate::core::registry::Registry; +use crate::core::{ManifestDependency, Summary}; +use crate::resolver::algorithm::in_memory_index::{InMemoryIndex, VersionsResponse}; +use crate::resolver::algorithm::provider::DependencyProviderError; +use futures::{FutureExt, StreamExt}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +#[derive(Default)] +pub(crate) struct ResolverState { + pub(crate) index: InMemoryIndex, +} + +impl ResolverState { + pub(crate) async fn fetch( + self: Arc, + registry: &dyn Registry, + request_stream: mpsc::Receiver, + ) -> Result<(), DependencyProviderError> { + let mut response_stream = ReceiverStream::new(request_stream) + .map(|request| self.process_request(request, registry).boxed_local()) + // Allow as many futures as possible to start in the background. + // Backpressure is provided by at a more granular level by `DistributionDatabase` + // and `SourceDispatch`, as well as the bounded request channel. + .buffer_unordered(usize::MAX); + + while let Some(response) = response_stream.next().await { + match response? { + Some(Response::Package(package, summaries)) => { + self.index + .packages() + .done(package, Arc::new(VersionsResponse::Found(summaries))); + } + None => {} + } + } + Ok(()) + } + + async fn process_request<'a>( + &self, + request: Request, + registry: &dyn Registry, + ) -> Result, DependencyProviderError> { + match request { + Request::Package(dependency) => { + self.index.packages().register(dependency.clone()); + let summaries = registry.query(&dependency).await?; + Ok(Some(Response::Package(dependency, summaries))) + } + } + } +} + +#[derive(Debug)] +pub(crate) enum Request { + Package(ManifestDependency), +} + +pub(crate) enum Response { + Package(ManifestDependency, Vec), +} diff --git a/scarb/src/resolver/mod.rs b/scarb/src/resolver/mod.rs index 111105890..f6a9ce3bd 100644 --- a/scarb/src/resolver/mod.rs +++ b/scarb/src/resolver/mod.rs @@ -37,7 +37,7 @@ pub async fn resolve( let algo_primitive = env::var("SCARB_UNSTABLE_PRIMITIVE_RESOLVER") .ok() .map(|var| var.as_str() == "true") - .unwrap_or(true); + .unwrap_or_default(); if algo_primitive { primitive::resolve(summaries, registry, lockfile).await } else { @@ -151,9 +151,9 @@ mod tests { #[test] fn single_fixed_dep() { check( - registry![("foo v1.0.0", []),], - &[deps![("foo", "=1.0.0")]], - Ok(pkgs!["foo v1.0.0"]), + registry![("foo v1.0.0-rc.0", []),], + &[deps![("foo", "=1.0.0-rc.0")]], + Ok(pkgs!["foo v1.0.0-rc.0"]), ) } @@ -262,20 +262,7 @@ mod tests { ("baz v1.0.0", []), ], &[deps![("foo", "*")]], - // TODO(#2): Expected result is commented out. - // Ok(pkgs![ - // "bar v1.0.0", - // "baz v1.0.0", - // "foo v1.0.0" - // ]), - Err(indoc! {" - Version solving failed: - - bar v2.0.0 cannot use baz v1.0.0, because bar requires baz ^2.0.0 - - Scarb does not have real version solving algorithm yet. - Perhaps in the future this conflict could be resolved, but currently, - please upgrade your dependencies to use latest versions of their dependencies. - "}), + Ok(pkgs!["bar v1.0.0", "baz v1.0.0", "foo v1.0.0"]), ) } @@ -294,20 +281,7 @@ mod tests { ("baz v2.1.0", []), ], &[deps![("bar", "~1.1.0"), ("foo", "~2.7")]], - // TODO(#2): Expected result is commented out. - // Ok(pkgs![ - // "bar v1.1.1", - // "baz v1.7.1", - // "foo v2.7.0" - // ]), - Err(indoc! {" - Version solving failed: - - foo v2.7.0 cannot use baz v2.1.0, because foo requires baz ~1.7.1 - - Scarb does not have real version solving algorithm yet. - Perhaps in the future this conflict could be resolved, but currently, - please upgrade your dependencies to use latest versions of their dependencies. - "}), + Ok(pkgs!["bar v1.1.1", "baz v1.7.1", "foo v2.7.0"]), ) } @@ -348,12 +322,10 @@ mod tests { ], &[deps![("top1", "1"), ("top2", "1")]], Err(indoc! {" - Version solving failed: - - top2 v1.0.0 cannot use foo v1.0.0, because top2 requires foo ^2.0.0 - - Scarb does not have real version solving algorithm yet. - Perhaps in the future this conflict could be resolved, but currently, - please upgrade your dependencies to use latest versions of their dependencies. + version solving failed: + Because there is no version of top1 in >1.0.0, <2.0.0 and top1 1.0.0 depends on foo >=1.0.0, <2.0.0, top1 >=1.0.0, <2.0.0 depends on foo >=1.0.0, <2.0.0. + And because top2 1.0.0 depends on foo >=2.0.0, <3.0.0 and there is no version of top2 in >1.0.0, <2.0.0, top1 >=1.0.0, <2.0.0, top2 >=1.0.0, <2.0.0 are incompatible. + And because root_1 1.0.0 depends on top1 >=1.0.0, <2.0.0 and root_1 1.0.0 depends on top2 >=1.0.0, <2.0.0, root_1 1.0.0 is forbidden. "}), ) } @@ -372,7 +344,7 @@ mod tests { check( registry![("foo v2.0.0", []),], &[deps![("foo", "1.0.0")]], - Err(r#"cannot find package foo"#), + Err(r#"cannot get dependencies of `root_1@1.0.0`"#), ) } @@ -397,7 +369,7 @@ mod tests { ("b v3.8.14", []), ], &[deps![("a", "~3.6"), ("b", "~3.6")]], - Err(r#"cannot find package a"#), + Err(r#"cannot get dependencies of `root_1@1.0.0`"#), ) } @@ -417,7 +389,7 @@ mod tests { ("b v3.8.5", [("d", "2.9.0")]), ], &[deps![("a", "~3.6"), ("c", "~1.1"), ("b", "~3.6")]], - Err(r#"cannot find package a"#), + Err(r#"cannot get dependencies of `root_1@1.0.0`"#), ) } @@ -440,7 +412,7 @@ mod tests { ), ], &[deps![("e", "~1.0"), ("a", "~3.7"), ("b", "~3.7")]], - Err(r#"cannot find package e"#), + Err(r#"cannot get dependencies of `root_1@1.0.0`"#), ) } @@ -541,7 +513,7 @@ mod tests { registry![("foo v1.0.0", []),], &[deps![("foo", "2.0.0"),]], locks![("foo v1.0.0", [])], - Err("cannot find package foo"), + Err("cannot get dependencies of `root_1@1.0.0`"), ); } diff --git a/scarb/src/resolver/primitive.rs b/scarb/src/resolver/primitive.rs index 6e77327ba..1cec3ff5c 100644 --- a/scarb/src/resolver/primitive.rs +++ b/scarb/src/resolver/primitive.rs @@ -16,7 +16,6 @@ pub async fn resolve( registry: &dyn Registry, lockfile: Lockfile, ) -> anyhow::Result { - // TODO(#2): This is very bad, use PubGrub here. let mut graph = DiGraphMap::::new(); let main_packages = summaries diff --git a/scarb/tests/add.rs b/scarb/tests/add.rs index 33855d040..4e5d00814 100644 --- a/scarb/tests/add.rs +++ b/scarb/tests/add.rs @@ -213,7 +213,10 @@ fn runs_resolver_if_network_is_allowed() { "#}) .failure() .stdout_matches(indoc! {r#" - error: cannot find package dep + error: cannot get dependencies of `hello@1.0.0` + + Caused by: + cannot find package `dep ^1.0.0` "#}) .run(); } diff --git a/scarb/tests/git_source.rs b/scarb/tests/git_source.rs index d621cc5fe..eea8c4cea 100644 --- a/scarb/tests/git_source.rs +++ b/scarb/tests/git_source.rs @@ -165,6 +165,7 @@ fn fetch_with_nested_paths() { Scarb::quick_snapbox() .arg("fetch") .current_dir(&t) + .timeout(std::time::Duration::from_secs(60)) .assert() .success(); } diff --git a/scarb/tests/git_source_network.rs b/scarb/tests/git_source_network.rs index 5c498093c..44ea6e2bd 100644 --- a/scarb/tests/git_source_network.rs +++ b/scarb/tests/git_source_network.rs @@ -41,7 +41,8 @@ fn https_something_happens() { error: failed to clone into: [..] Caused by: - process did not exit successfully: exit [..]: 128 + 0: failed to clone into: [..] + 1: process did not exit successfully: exit [..]: 128 "#}); }); } @@ -76,7 +77,8 @@ fn ssh_something_happens() { error: failed to clone into: [..] Caused by: - process did not exit successfully: exit [..]: 128 + 0: failed to clone into: [..] + 1: process did not exit successfully: exit [..]: 128 "#}); }); } diff --git a/scarb/tests/http_registry.rs b/scarb/tests/http_registry.rs index d21a019cc..9d9f41037 100644 --- a/scarb/tests/http_registry.rs +++ b/scarb/tests/http_registry.rs @@ -40,7 +40,7 @@ fn usage() { [..] Downloading bar v1.0.0 ([..]) "#}); - let expected = expect![[" + let expected = expect![[r#" GET /api/v1/index/config.json accept: */* accept-encoding: gzip, br, deflate @@ -71,6 +71,19 @@ fn usage() { ### + GET /index/3/b/bar.json + accept: */* + accept-encoding: gzip, br, deflate + host: ... + if-none-match: ... + user-agent: ... + + 304 Not Modified + content-length: 0 + etag: ... + + ### + GET /bar-1.0.0.tar.zst accept: */* accept-encoding: gzip, br, deflate @@ -83,7 +96,7 @@ fn usage() { content-type: application/octet-stream etag: ... last-modified: ... - "]]; + "#]]; expected.assert_eq(®istry.logs()); } @@ -117,7 +130,7 @@ fn publish_verified() { [..] Downloading bar v1.0.0 ([..]) "#}); - let expected = expect![[" + let expected = expect![[r#" GET /api/v1/index/config.json accept: */* accept-encoding: gzip, br, deflate @@ -148,6 +161,19 @@ fn publish_verified() { ### + GET /index/3/b/bar.json + accept: */* + accept-encoding: gzip, br, deflate + host: ... + if-none-match: ... + user-agent: ... + + 304 Not Modified + content-length: 0 + etag: ... + + ### + GET /bar-1.0.0.tar.zst accept: */* accept-encoding: gzip, br, deflate @@ -160,7 +186,7 @@ fn publish_verified() { content-type: application/octet-stream etag: ... last-modified: ... - "]]; + "#]]; expected.assert_eq(®istry.logs()); } @@ -194,7 +220,8 @@ fn not_found() { error: failed to lookup for `baz ^1 (registry+http://[..])` in registry: registry+http://[..] Caused by: - package not found in registry: baz ^1 (registry+http://[..]) + 0: failed to lookup for `baz ^1 (registry+http://[..])` in registry: registry+http://[..] + 1: package not found in registry: baz ^1 (registry+http://[..]) "#}); let expected = expect![[" @@ -248,8 +275,9 @@ fn missing_config_json() { error: failed to lookup for `baz ^1 (registry+http://[..])` in registry: registry+http://[..] Caused by: - 0: failed to fetch registry config - 1: HTTP status client error (404 Not Found) for url (http://[..]/config.json) + 0: failed to lookup for `baz ^1 (registry+http://[..])` in registry: registry+http://[..] + 1: failed to fetch registry config + 2: HTTP status client error (404 Not Found) for url (http://[..]/config.json) "#}); let expected = expect![[" @@ -349,6 +377,19 @@ fn caching() { ### + GET /index/3/b/bar.json + accept: */* + accept-encoding: gzip, br, deflate + host: ... + if-none-match: ... + user-agent: ... + + 304 Not Modified + content-length: 0 + etag: ... + + ### + GET /bar-1.0.0.tar.zst accept: */* accept-encoding: gzip, br, deflate @@ -387,6 +428,32 @@ fn caching() { 304 Not Modified content-length: 0 etag: ... + + ### + + GET /index/3/b/bar.json + accept: */* + accept-encoding: gzip, br, deflate + host: ... + if-none-match: ... + user-agent: ... + + 304 Not Modified + content-length: 0 + etag: ... + + ### + + GET /index/3/b/bar.json + accept: */* + accept-encoding: gzip, br, deflate + host: ... + if-none-match: ... + user-agent: ... + + 304 Not Modified + content-length: 0 + etag: ... "#]]; expected.assert_eq(®istry.logs()); } diff --git a/scarb/tests/local_registry.rs b/scarb/tests/local_registry.rs index 27b789c45..c914e17f9 100644 --- a/scarb/tests/local_registry.rs +++ b/scarb/tests/local_registry.rs @@ -66,7 +66,8 @@ fn not_found() { error: failed to lookup for `baz ^1 (registry+file://[..])` in registry: registry+file://[..] Caused by: - package not found in registry: baz ^1 (registry+file://[..]) + 0: failed to lookup for `baz ^1 (registry+file://[..])` in registry: registry+file://[..] + 1: package not found in registry: baz ^1 (registry+file://[..]) "#}); } @@ -93,7 +94,8 @@ fn empty_registry() { error: failed to lookup for `baz ^1 (registry+file://[..])` in registry: registry+file://[..] Caused by: - package not found in registry: baz ^1 (registry+file://[..]) + 0: failed to lookup for `baz ^1 (registry+file://[..])` in registry: registry+file://[..] + 1: package not found in registry: baz ^1 (registry+file://[..]) "#}); } @@ -120,7 +122,8 @@ fn url_pointing_to_file() { error: failed to load source: registry+file://[..] Caused by: - local registry path is not a directory: [..] + 0: failed to load source: registry+file://[..] + 1: local registry path is not a directory: [..] "#}); // Prevent the temp directory from being deleted until this point. diff --git a/scarb/tests/resolver_with_git.rs b/scarb/tests/resolver_with_git.rs index 9e36d5f1d..7fa688b3d 100644 --- a/scarb/tests/resolver_with_git.rs +++ b/scarb/tests/resolver_with_git.rs @@ -1,10 +1,10 @@ use assert_fs::prelude::*; use assert_fs::TempDir; use indoc::indoc; - use scarb_test_support::command::Scarb; use scarb_test_support::gitx; use scarb_test_support::project_builder::{DepBuilder, ProjectBuilder}; +use snapbox::assert_matches; #[test] fn valid_triangle() { @@ -33,15 +33,34 @@ fn valid_triangle() { .dep("proxy", &proxy) .build(&t); - Scarb::quick_snapbox() + let output = Scarb::quick_snapbox() .arg("fetch") .current_dir(&t) - .assert() - .success() - .stdout_matches(indoc! {r#" - [..] Updating git repository file://[..]/culprit - [..] Updating git repository file://[..]/proxy - "#}); + .output() + .unwrap(); + + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + assert!( + output.status.success(), + "output is not success:\n{}", + stderr.clone() + ); + + let output = String::from_utf8_lossy(&output.stdout).to_string(); + assert_matches( + indoc! {r#" + [..] Updating git repository file://[..] + [..] Updating git repository file://[..] + "#}, + &output, + ); + + assert!( + // Order is not assured. + output.contains("/proxy") && output.contains("/culprit"), + "{}", + stderr + ); } #[test] @@ -73,18 +92,33 @@ fn two_revs_of_same_dep() { .dep("proxy", &proxy) .build(&t); - Scarb::quick_snapbox() + let output = Scarb::quick_snapbox() .arg("fetch") .current_dir(&t) - .assert() - .failure() - .stdout_matches(indoc! {r#" - [..] Updating git repository file://[..]/culprit - [..] Updating git repository file://[..]/culprit - error: found dependencies on the same package `culprit` coming from incompatible sources: - source 1: git+file://[..]/culprit#[..] - source 2: git+file://[..]/culprit?branch=branchy#[..] - "#}); + .output() + .unwrap(); + + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + assert!(!output.status.success(), "{}", stderr.clone()); + + let output = String::from_utf8_lossy(&output.stdout).to_string(); + assert_matches( + indoc! {r#" + [..] Updating git repository file://[..]/culprit + [..] Updating git repository file://[..]/culprit + error: found dependencies on the same package `culprit` coming from incompatible sources: + source 1: git+file://[..]/culprit[..] + source 2: git+file://[..]/culprit[..] + "#}, + &output, + ); + + assert!( + // Order is not assured. + output.contains("culprit?branch=branchy#") && output.contains("culprit#"), + "{}", + stderr + ); } #[test] @@ -125,18 +159,40 @@ fn two_revs_of_same_dep_diamond() { .dep("dep2", &dep2) .build(&t); - Scarb::quick_snapbox() + let output = Scarb::quick_snapbox() .arg("fetch") .current_dir(&t) - .assert() - .failure() - .stdout_matches(indoc! {r#" - [..] Updating git repository file://[..]/dep1 - [..] Updating git repository file://[..]/dep2 - [..] Updating git repository file://[..]/culprit - [..] Updating git repository file://[..]/culprit + .output() + .unwrap(); + + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + assert!(!output.status.success(), "{}", stderr.clone()); + + let output = String::from_utf8_lossy(&output.stdout).to_string(); + assert_matches( + indoc! {r#" + [..] Updating git repository file://[..] + [..] Updating git repository file://[..] + [..] Updating git repository file://[..] + [..] Updating git repository file://[..] error: found dependencies on the same package `culprit` coming from incompatible sources: - source 1: git+file://[..]/culprit#[..] - source 2: git+file://[..]/culprit?branch=branchy#[..] - "#}); + source 1: git+file://[..]/culprit[..] + source 2: git+file://[..]/culprit[..] + "#}, + &output, + ); + + assert!( + // Order is not assured. + output.contains("/dep1") && output.contains("/dep2") && output.contains("/culprit"), + "{}", + stderr.clone() + ); + + assert!( + // Order is not assured. + output.contains("/culprit?branch=branchy#") && output.contains("/culprit#"), + "{}", + stderr + ); }