From 32d3b078c10e68cc7a45c23f674f1aac0c1679af Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 13:01:05 +0200 Subject: [PATCH 01/10] refactor: make solver generic --- crates/rattler-bin/src/commands/create.rs | 32 ++++------- crates/rattler_solve/src/lib.rs | 56 +++++++++---------- crates/rattler_solve/src/libsolv/input.rs | 6 +- crates/rattler_solve/src/libsolv/mod.rs | 39 +++++++++---- crates/rattler_solve/src/libsolv/output.rs | 6 +- crates/rattler_solve/src/libsolv_rs/input.rs | 4 +- crates/rattler_solve/src/libsolv_rs/mod.rs | 28 ++++++---- crates/rattler_solve/src/libsolv_rs/output.rs | 6 +- crates/rattler_solve/src/solver_backend.rs | 43 +++++++++++++- 9 files changed, 136 insertions(+), 84 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 572fc5414..aac2bc465 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -11,11 +11,9 @@ use rattler_conda_types::{ RepoDataRecord, Version, }; use rattler_networking::{AuthenticatedClient, AuthenticationStorage}; -use rattler_repodata_gateway::fetch::{ - CacheResult, DownloadProgress, FetchRepoDataError, FetchRepoDataOptions, -}; +use rattler_repodata_gateway::fetch::{DownloadProgress, FetchRepoDataError, FetchRepoDataOptions}; use rattler_repodata_gateway::sparse::SparseRepoData; -use rattler_solve::{LibsolvRepoData, LibsolvRsRepoData, SolverBackend, SolverTask}; +use rattler_solve::{SolverBackend, SolverTask}; use reqwest::{Client, StatusCode}; use std::{ borrow::Cow, @@ -201,31 +199,21 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { .map(|record| record.repodata_record.clone()) .collect(); + let solver_task = SolverTask { + available_packages: &repodatas, + locked_packages, + virtual_packages, + specs, + pinned_packages: Vec::new(), + }; + // Next, use a solver to solve this specific problem. This provides us with all the operations // we need to apply to our environment to bring it up to date. let use_libsolv_rs = opt.use_experimental_libsolv_rs; let required_packages = wrap_in_progress("solving", move || { if use_libsolv_rs { - let solver_task = SolverTask { - available_packages: repodatas - .iter() - .map(|records| LibsolvRsRepoData::from_records(records)), - locked_packages, - virtual_packages, - specs, - pinned_packages: Vec::new(), - }; rattler_solve::LibsolvRsBackend.solve(solver_task) } else { - let solver_task = SolverTask { - available_packages: repodatas - .iter() - .map(|records| LibsolvRepoData::from_records(records)), - locked_packages, - virtual_packages, - specs, - pinned_packages: Vec::new(), - }; rattler_solve::LibsolvBackend.solve(solver_task) } })?; diff --git a/crates/rattler_solve/src/lib.rs b/crates/rattler_solve/src/lib.rs index 18712a14a..ae37ad991 100644 --- a/crates/rattler_solve/src/lib.rs +++ b/crates/rattler_solve/src/lib.rs @@ -84,9 +84,7 @@ pub struct SolverTask { #[cfg(test)] mod test_libsolv { use crate::libsolv::LibsolvBackend; - use crate::{ - LibsolvRepoData, LibsolvRsBackend, LibsolvRsRepoData, SolveError, SolverBackend, SolverTask, - }; + use crate::{LibsolvRepoData, LibsolvRsBackend, SolveError, SolverBackend, SolverTask}; use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, NoArchType, PackageRecord, RepoData, RepoDataRecord, Version, @@ -187,7 +185,7 @@ mod test_libsolv { } } - fn solve_real_world(specs: Vec<&str>) -> (Vec, Vec) { + fn solve_real_world(specs: Vec<&str>) -> Vec { let specs = specs .iter() .map(|s| MatchSpec::from_str(s).unwrap()) @@ -206,9 +204,7 @@ mod test_libsolv { SparseRepoData::load_records_recursive(&sparse_repo_datas, names).unwrap(); let solver_task = SolverTask { - available_packages: available_packages - .iter() - .map(|records| LibsolvRepoData::from_records(records)), + available_packages: &available_packages, specs: specs.clone(), locked_packages: Default::default(), pinned_packages: Default::default(), @@ -217,17 +213,6 @@ mod test_libsolv { let pkgs1 = LibsolvBackend.solve(solver_task).unwrap(); - let solver_task = SolverTask { - available_packages: available_packages - .iter() - .map(|records| LibsolvRsRepoData::from_records(records)), - specs, - locked_packages: Default::default(), - pinned_packages: Default::default(), - virtual_packages: Default::default(), - }; - let pkgs2 = LibsolvRsBackend.solve(solver_task).unwrap(); - let extract_pkgs = |records: Vec| { let mut pkgs = records .into_iter() @@ -247,21 +232,36 @@ mod test_libsolv { pkgs }; - (extract_pkgs(pkgs1), extract_pkgs(pkgs2)) + extract_pkgs(pkgs1) } + #[cfg(feature = "libsolv")] #[test] - fn test_solve_tensorboard() { - let (libsolv_pkgs, libsolv_rs_pkgs) = - solve_real_world(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); + fn test_solve_tensorboard_libsolv_sys() { + let libsolv_pkgs = + solve_real_world::(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); insta::assert_yaml_snapshot!("solve_tensorboard_libsolv", libsolv_pkgs); - insta::assert_yaml_snapshot!("solve_tensorboard_libsolv_rs", libsolv_rs_pkgs); } + #[cfg(feature = "libsolv")] #[test] - fn test_solve_python() { - let (libsolv_pkgs, libsolv_rs_pkgs) = solve_real_world(vec!["python=3.9"]); + fn test_solve_python_libsolv_sys() { + let libsolv_pkgs = solve_real_world::(vec!["python=3.9"]); insta::assert_yaml_snapshot!("solve_python_libsolv", libsolv_pkgs); + } + + #[cfg(feature = "libsolv-rs")] + #[test] + fn test_solve_tensorboard_libsolv_rs() { + let libsolv_rs_pkgs = + solve_real_world::(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); + insta::assert_yaml_snapshot!("solve_tensorboard_libsolv_rs", libsolv_rs_pkgs); + } + + #[cfg(feature = "libsolv-rs")] + #[test] + fn test_solve_python_libsolv_rs() { + let libsolv_rs_pkgs = solve_real_world::(vec!["python=3.9"]); insta::assert_yaml_snapshot!("solve_python_libsolv_rs", libsolv_rs_pkgs); } @@ -569,7 +569,7 @@ mod test_libsolv { solv_file: Some(&cached_repo_data), } } else { - LibsolvRepoData::from_records(&repo_data) + LibsolvRepoData::from_iter(&repo_data) }; let specs: Vec<_> = match_specs @@ -580,7 +580,7 @@ mod test_libsolv { let task = SolverTask { locked_packages: installed_packages.clone(), virtual_packages: virtual_packages.clone(), - available_packages: vec![libsolv_repodata].into_iter(), + available_packages: [libsolv_repodata], specs: specs.clone(), pinned_packages: Vec::new(), }; @@ -589,7 +589,7 @@ mod test_libsolv { let task = SolverTask { locked_packages: installed_packages, virtual_packages, - available_packages: vec![LibsolvRsRepoData::from_records(&repo_data)].into_iter(), + available_packages: [&repo_data], specs, pinned_packages: Vec::new(), }; diff --git a/crates/rattler_solve/src/libsolv/input.rs b/crates/rattler_solve/src/libsolv/input.rs index 247891bed..831f38e05 100644 --- a/crates/rattler_solve/src/libsolv/input.rs +++ b/crates/rattler_solve/src/libsolv/input.rs @@ -38,10 +38,10 @@ pub fn add_solv_file(pool: &Pool, repo: &Repo, solv_bytes: &LibcByteSlice) { /// Adds [`RepoDataRecord`] to `repo` /// /// Panics if the repo does not belong to the pool -pub fn add_repodata_records( +pub fn add_repodata_records<'a>( pool: &Pool, repo: &Repo, - repo_datas: &[RepoDataRecord], + repo_datas: impl IntoIterator, ) -> Vec { // Sanity check repo.ensure_belongs_to_pool(pool); @@ -70,7 +70,7 @@ pub fn add_repodata_records( let data = repo.add_repodata(); let mut solvable_ids = Vec::new(); - for (repo_data_index, repo_data) in repo_datas.iter().enumerate() { + for (repo_data_index, repo_data) in repo_datas.into_iter().enumerate() { // Create a solvable for the package let solvable_id = match add_or_reuse_solvable(pool, repo, &data, &mut package_to_type, repo_data) { diff --git a/crates/rattler_solve/src/libsolv/mod.rs b/crates/rattler_solve/src/libsolv/mod.rs index dbb64e89e..b8d5aaa67 100644 --- a/crates/rattler_solve/src/libsolv/mod.rs +++ b/crates/rattler_solve/src/libsolv/mod.rs @@ -1,3 +1,4 @@ +use crate::solver_backend::{IntoRepoData, SolverRepoData}; use crate::{SolveError, SolverBackend, SolverTask}; pub use input::cache_repodata; use input::{add_repodata_records, add_solv_file, add_virtual_packages}; @@ -23,22 +24,34 @@ mod wrapper; #[derive(Clone)] pub struct LibsolvRepoData<'a> { /// The actual records after parsing `repodata.json` - pub records: &'a [RepoDataRecord], + pub records: Vec<&'a RepoDataRecord>, /// The in-memory .solv file built from the records (if available) pub solv_file: Option<&'a LibcByteSlice>, } -impl LibsolvRepoData<'_> { - /// Constructs a new `LibsolvRepoData` without a corresponding .solv file - pub fn from_records(records: &[RepoDataRecord]) -> LibsolvRepoData { - LibsolvRepoData { - records, +impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRepoData<'a> { + fn from_iter>(iter: T) -> Self { + Self { + records: Vec::from_iter(iter), solv_file: None, } } } +impl<'a> LibsolvRepoData<'a> { + /// Constructs a new `LibsolvRsRepoData` + #[deprecated(since = "0.7.0", note = "use From::from instead")] + pub fn from_records(records: impl Into>) -> Self { + Self { + records: records.into(), + solv_file: None, + } + } +} + +impl<'a> SolverRepoData<'a> for LibsolvRepoData<'a> {} + /// Convenience method that converts a string reference to a CString, replacing NUL characters with /// whitespace (` `) fn c_string>(str: T) -> CString { @@ -66,7 +79,11 @@ pub struct LibsolvBackend; impl SolverBackend for LibsolvBackend { type RepoData<'a> = LibsolvRepoData<'a>; - fn solve<'a, TAvailablePackagesIterator: Iterator>>( + fn solve< + 'a, + R: IntoRepoData<'a, Self::RepoData<'a>>, + TAvailablePackagesIterator: IntoIterator, + >( &mut self, task: SolverTask, ) -> Result, SolveError> { @@ -89,7 +106,7 @@ impl SolverBackend for LibsolvBackend { // Create repos for all channel + platform combinations let mut repo_mapping = HashMap::new(); let mut all_repodata_records = Vec::new(); - for repodata in task.available_packages { + for repodata in task.available_packages.into_iter().map(IntoRepoData::into) { if repodata.records.is_empty() { continue; } @@ -100,7 +117,7 @@ impl SolverBackend for LibsolvBackend { if let Some(solv_file) = repodata.solv_file { add_solv_file(&pool, &repo, solv_file); } else { - add_repodata_records(&pool, &repo, repodata.records); + add_repodata_records(&pool, &repo, repodata.records.iter().copied()); } // Keep our own info about repodata_records @@ -117,7 +134,7 @@ impl SolverBackend for LibsolvBackend { // Also add the installed records to the repodata repo_mapping.insert(repo.id(), repo_mapping.len()); - all_repodata_records.push(&task.locked_packages); + all_repodata_records.push(task.locked_packages.iter().collect()); // Create a special pool for records that are pinned and cannot be changed. let repo = Repo::new(&pool, "pinned"); @@ -125,7 +142,7 @@ impl SolverBackend for LibsolvBackend { // Also add the installed records to the repodata repo_mapping.insert(repo.id(), repo_mapping.len()); - all_repodata_records.push(&task.pinned_packages); + all_repodata_records.push(task.pinned_packages.iter().collect()); // Create datastructures for solving pool.create_whatprovides(); diff --git a/crates/rattler_solve/src/libsolv/output.rs b/crates/rattler_solve/src/libsolv/output.rs index 16113e85b..0f8747a82 100644 --- a/crates/rattler_solve/src/libsolv/output.rs +++ b/crates/rattler_solve/src/libsolv/output.rs @@ -13,11 +13,11 @@ use std::collections::HashMap; /// /// If the transaction contains libsolv operations that are not "install" an error is returned /// containing their ids. -pub fn get_required_packages( +pub fn get_required_packages<'a>( pool: &Pool, repo_mapping: &HashMap, transaction: &Transaction, - repodata_records: &[&[RepoDataRecord]], + repodata_records: &[Vec<&'a RepoDataRecord>], ) -> Result, Vec> { let mut required_packages = Vec::new(); let mut unsupported_operations = Vec::new(); @@ -35,7 +35,7 @@ pub fn get_required_packages( // Retrieve the repodata record corresponding to this solvable let (repo_index, solvable_index) = get_solvable_indexes(pool, repo_mapping, solvable_index_id, id); - let repodata_record = &repodata_records[repo_index][solvable_index]; + let repodata_record = repodata_records[repo_index][solvable_index]; match transaction_type as u32 { ffi::SOLVER_TRANSACTION_INSTALL => { diff --git a/crates/rattler_solve/src/libsolv_rs/input.rs b/crates/rattler_solve/src/libsolv_rs/input.rs index c5030296a..96e095e0c 100644 --- a/crates/rattler_solve/src/libsolv_rs/input.rs +++ b/crates/rattler_solve/src/libsolv_rs/input.rs @@ -13,13 +13,13 @@ use std::collections::HashMap; pub fn add_repodata_records<'a>( pool: &mut Pool<'a>, repo_id: RepoId, - repo_datas: &'a [RepoDataRecord], + repo_datas: impl IntoIterator, ) -> Vec { // Keeps a mapping from packages added to the repo to the type and solvable let mut package_to_type: HashMap<&str, (ArchiveType, SolvableId)> = HashMap::new(); let mut solvable_ids = Vec::new(); - for (repo_data_index, repo_data) in repo_datas.iter().enumerate() { + for (repo_data_index, repo_data) in repo_datas.into_iter().enumerate() { // Create a solvable for the package let solvable_id = match add_or_reuse_solvable(pool, repo_id, &mut package_to_type, repo_data) { diff --git a/crates/rattler_solve/src/libsolv_rs/mod.rs b/crates/rattler_solve/src/libsolv_rs/mod.rs index 4137cec4d..41befeec6 100644 --- a/crates/rattler_solve/src/libsolv_rs/mod.rs +++ b/crates/rattler_solve/src/libsolv_rs/mod.rs @@ -1,3 +1,4 @@ +use crate::solver_backend::{IntoRepoData, SolverRepoData}; use crate::{SolveError, SolverBackend, SolverTask}; use input::{add_repodata_records, add_virtual_packages}; use libsolv_rs::{Pool, SolveJobs, Solver}; @@ -13,23 +14,30 @@ mod output; #[derive(Clone)] pub struct LibsolvRsRepoData<'a> { /// The actual records after parsing `repodata.json` - pub records: &'a [RepoDataRecord], + pub records: Vec<&'a RepoDataRecord>, } -impl LibsolvRsRepoData<'_> { - /// Constructs a new `LibsolvRsRepoData` - pub fn from_records(records: &[RepoDataRecord]) -> LibsolvRsRepoData { - LibsolvRsRepoData { records } +impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRsRepoData<'a> { + fn from_iter>(iter: T) -> Self { + Self { + records: Vec::from_iter(iter), + } } } +impl<'a> SolverRepoData<'a> for LibsolvRsRepoData<'a> {} + /// A [`SolverBackend`] implemented using the `libsolv` library pub struct LibsolvRsBackend; impl SolverBackend for LibsolvRsBackend { type RepoData<'a> = LibsolvRsRepoData<'a>; - fn solve<'a, TAvailablePackagesIterator: Iterator>>( + fn solve< + 'a, + R: IntoRepoData<'a, Self::RepoData<'a>>, + TAvailablePackagesIterator: IntoIterator, + >( &mut self, task: SolverTask, ) -> Result, SolveError> { @@ -43,13 +51,13 @@ impl SolverBackend for LibsolvRsBackend { // Create repos for all channel + platform combinations let mut repo_mapping = HashMap::new(); let mut all_repodata_records = Vec::new(); - for repodata in task.available_packages { + for repodata in task.available_packages.into_iter().map(IntoRepoData::into) { if repodata.records.is_empty() { continue; } let repo_id = pool.new_repo(); - add_repodata_records(&mut pool, repo_id, repodata.records); + add_repodata_records(&mut pool, repo_id, repodata.records.iter().copied()); // Keep our own info about repodata_records repo_mapping.insert(repo_id, repo_mapping.len()); @@ -62,7 +70,7 @@ impl SolverBackend for LibsolvRsBackend { // Also add the installed records to the repodata repo_mapping.insert(repo_id, repo_mapping.len()); - all_repodata_records.push(&task.locked_packages); + all_repodata_records.push(task.locked_packages.iter().collect()); // Create a special pool for records that are pinned and cannot be changed. let repo_id = pool.new_repo(); @@ -70,7 +78,7 @@ impl SolverBackend for LibsolvRsBackend { // Also add the installed records to the repodata repo_mapping.insert(repo_id, repo_mapping.len()); - all_repodata_records.push(&task.pinned_packages); + all_repodata_records.push(task.pinned_packages.iter().collect()); // Add matchspec to the queue let mut goal = SolveJobs::default(); diff --git a/crates/rattler_solve/src/libsolv_rs/output.rs b/crates/rattler_solve/src/libsolv_rs/output.rs index c3418207a..c26ee2f95 100644 --- a/crates/rattler_solve/src/libsolv_rs/output.rs +++ b/crates/rattler_solve/src/libsolv_rs/output.rs @@ -9,11 +9,11 @@ use std::collections::HashMap; /// /// If the transaction contains libsolv operations that are not "install" an error is returned /// containing their ids. -pub fn get_required_packages( +pub fn get_required_packages<'a>( pool: &Pool, repo_mapping: &HashMap, transaction: &Transaction, - repodata_records: &[&[RepoDataRecord]], + repodata_records: &[Vec<&'a RepoDataRecord>], ) -> Vec { let mut required_packages = Vec::new(); @@ -22,7 +22,7 @@ pub fn get_required_packages( // // Packages without indexes are virtual and can be ignored if let Some((repo_index, solvable_index)) = get_solvable_indexes(pool, repo_mapping, id) { - let repodata_record = &repodata_records[repo_index][solvable_index]; + let repodata_record = repodata_records[repo_index][solvable_index]; required_packages.push(repodata_record.clone()); } } diff --git a/crates/rattler_solve/src/solver_backend.rs b/crates/rattler_solve/src/solver_backend.rs index 90d0a0eb0..d278c4168 100644 --- a/crates/rattler_solve/src/solver_backend.rs +++ b/crates/rattler_solve/src/solver_backend.rs @@ -1,14 +1,53 @@ use crate::{SolveError, SolverTask}; use rattler_conda_types::RepoDataRecord; +/// A representation of a collection of [`RepoDataRecord`] usable by a [`SolverBackend`] +/// implementation. +/// +/// Some solvers might be able to cache the collection between different runs of the solver which +/// could potentially eliminate some overhead. This trait enables creating a representation of the +/// repodata that is most suitable for a specific backend. +/// +/// Some solvers may add additional functionality to their specific implementation that enables +/// caching the repodata to disk in an efficient way (see [`crate::libsolv::LibSolvRepoData`] for +/// an example). +pub trait SolverRepoData<'a>: FromIterator<&'a RepoDataRecord> {} + +/// Defines the ability to convert a type into [`SolverRepoData`]. +pub trait IntoRepoData<'a, S: SolverRepoData<'a>> { + fn into(self) -> S; +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a Vec { + fn into(self) -> S { + S::from_iter(self.iter()) + } +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a [RepoDataRecord] { + fn into(self) -> S { + S::from_iter(self.iter()) + } +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for S { + fn into(self) -> S { + self + } +} + /// Represents a solver backend, capable of solving [`SolverTask`]s pub trait SolverBackend { /// The repo data associated to a channel and platform combination - type RepoData<'a>; + type RepoData<'a>: SolverRepoData<'a>; /// Resolve the dependencies and return the [`RepoDataRecord`]s that should be present in the /// environment. - fn solve<'a, TAvailablePackagesIterator: Iterator>>( + fn solve< + 'a, + R: IntoRepoData<'a, Self::RepoData<'a>>, + TAvailablePackagesIterator: IntoIterator, + >( &mut self, task: SolverTask, ) -> Result, SolveError>; From c4f748d199842f5a93770af02c8800d0e56fb63b Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 14:55:33 +0200 Subject: [PATCH 02/10] fix: tests and build and clippy --- crates/rattler-bin/src/commands/create.rs | 4 +++- crates/rattler_solve/src/libsolv/output.rs | 4 ++-- crates/rattler_solve/src/libsolv_rs/output.rs | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index aac2bc465..0c5b86b52 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -11,7 +11,9 @@ use rattler_conda_types::{ RepoDataRecord, Version, }; use rattler_networking::{AuthenticatedClient, AuthenticationStorage}; -use rattler_repodata_gateway::fetch::{DownloadProgress, FetchRepoDataError, FetchRepoDataOptions}; +use rattler_repodata_gateway::fetch::{ + CacheResult, DownloadProgress, FetchRepoDataError, FetchRepoDataOptions, +}; use rattler_repodata_gateway::sparse::SparseRepoData; use rattler_solve::{SolverBackend, SolverTask}; use reqwest::{Client, StatusCode}; diff --git a/crates/rattler_solve/src/libsolv/output.rs b/crates/rattler_solve/src/libsolv/output.rs index 0f8747a82..68424b983 100644 --- a/crates/rattler_solve/src/libsolv/output.rs +++ b/crates/rattler_solve/src/libsolv/output.rs @@ -13,11 +13,11 @@ use std::collections::HashMap; /// /// If the transaction contains libsolv operations that are not "install" an error is returned /// containing their ids. -pub fn get_required_packages<'a>( +pub fn get_required_packages( pool: &Pool, repo_mapping: &HashMap, transaction: &Transaction, - repodata_records: &[Vec<&'a RepoDataRecord>], + repodata_records: &[Vec<&RepoDataRecord>], ) -> Result, Vec> { let mut required_packages = Vec::new(); let mut unsupported_operations = Vec::new(); diff --git a/crates/rattler_solve/src/libsolv_rs/output.rs b/crates/rattler_solve/src/libsolv_rs/output.rs index c26ee2f95..2ff9091de 100644 --- a/crates/rattler_solve/src/libsolv_rs/output.rs +++ b/crates/rattler_solve/src/libsolv_rs/output.rs @@ -9,11 +9,11 @@ use std::collections::HashMap; /// /// If the transaction contains libsolv operations that are not "install" an error is returned /// containing their ids. -pub fn get_required_packages<'a>( +pub fn get_required_packages( pool: &Pool, repo_mapping: &HashMap, transaction: &Transaction, - repodata_records: &[Vec<&'a RepoDataRecord>], + repodata_records: &[Vec<&RepoDataRecord>], ) -> Vec { let mut required_packages = Vec::new(); From 6060c1ee037671289634701d626033fd51dfcb1f Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 15:24:26 +0200 Subject: [PATCH 03/10] fix: unix tests --- crates/rattler_solve/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_solve/src/lib.rs b/crates/rattler_solve/src/lib.rs index ae37ad991..8d7ce3056 100644 --- a/crates/rattler_solve/src/lib.rs +++ b/crates/rattler_solve/src/lib.rs @@ -565,7 +565,7 @@ mod test_libsolv { #[cfg(target_family = "unix")] LibsolvRepoData { - records: repo_data.as_slice(), + records: repo_data.iter().collect(), solv_file: Some(&cached_repo_data), } } else { From a1d74c9c4fb4d263ec8edac6b69ca47a77f45aa6 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 15:37:41 +0200 Subject: [PATCH 04/10] Update crates/rattler_solve/src/libsolv/mod.rs --- crates/rattler_solve/src/libsolv/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_solve/src/libsolv/mod.rs b/crates/rattler_solve/src/libsolv/mod.rs index b8d5aaa67..68ff97638 100644 --- a/crates/rattler_solve/src/libsolv/mod.rs +++ b/crates/rattler_solve/src/libsolv/mod.rs @@ -41,7 +41,7 @@ impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRepoData<'a> { impl<'a> LibsolvRepoData<'a> { /// Constructs a new `LibsolvRsRepoData` - #[deprecated(since = "0.7.0", note = "use From::from instead")] + #[deprecated(since = "0.6.0", note = "use From::from instead")] pub fn from_records(records: impl Into>) -> Self { Self { records: records.into(), From 9ca4262318567db25dbbe6a153e52f9e186ea9db Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 17:09:12 +0200 Subject: [PATCH 05/10] refactor: move tests to integration tests --- crates/rattler-bin/Cargo.toml | 2 +- crates/rattler_solve/Cargo.toml | 5 +- crates/rattler_solve/src/lib.rs | 551 +----------------- crates/rattler_solve/src/libsolv/mod.rs | 1 + crates/rattler_solve/src/libsolv_rs/mod.rs | 1 + crates/rattler_solve/tests/backends.rs | 481 +++++++++++++++ ...solve_dummy_repo_install_non_existent.snap | 10 + ..._libsolv_rs__solve_python_libsolv_rs.snap} | 5 +- ...olv_rs__solve_tensorboard_libsolv_rs.snap} | 5 +- ...solve_dummy_repo_install_non_existent.snap | 10 + ...libsolv_sys__solve_python_libsolv_rs.snap} | 5 +- ...lv_sys__solve_tensorboard_libsolv_rs.snap} | 5 +- 12 files changed, 522 insertions(+), 559 deletions(-) create mode 100644 crates/rattler_solve/tests/backends.rs create mode 100644 crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_dummy_repo_install_non_existent.snap rename crates/rattler_solve/{src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv_rs.snap => tests/snapshots/backends__libsolv_rs__solve_python_libsolv_rs.snap} (80%) rename crates/rattler_solve/{src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv_rs.snap => tests/snapshots/backends__libsolv_rs__solve_tensorboard_libsolv_rs.snap} (93%) create mode 100644 crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_dummy_repo_install_non_existent.snap rename crates/rattler_solve/{src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv.snap => tests/snapshots/backends__libsolv_sys__solve_python_libsolv_rs.snap} (80%) rename crates/rattler_solve/{src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv.snap => tests/snapshots/backends__libsolv_sys__solve_tensorboard_libsolv_rs.snap} (93%) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 6bc4bbc08..4513cf906 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -32,7 +32,7 @@ rattler = { version = "0.5.0", path = "../rattler", default-features = false } rattler_networking = { version = "0.5.0", path = "../rattler_networking", default-features = false } rattler_conda_types = { version = "0.5.0", path = "../rattler_conda_types" } rattler_repodata_gateway = { version = "0.5.0", path = "../rattler_repodata_gateway", features = ["sparse"], default-features = false } -rattler_solve = { version = "0.5.0", path = "../rattler_solve", features = ["libsolv-rs"] } +rattler_solve = { version = "0.5.0", path = "../rattler_solve", features = ["libsolv_rs", "libsolv-sys"] } rattler_virtual_packages = { version = "0.5.0", path = "../rattler_virtual_packages" } reqwest = { version = "0.11.18", default-features = false } tokio = { version = "1.28.2", features = ["rt-multi-thread", "macros"] } diff --git a/crates/rattler_solve/Cargo.toml b/crates/rattler_solve/Cargo.toml index 091a96441..973555b69 100644 --- a/crates/rattler_solve/Cargo.toml +++ b/crates/rattler_solve/Cargo.toml @@ -32,6 +32,5 @@ serde_json = "1.0.96" url = "2.4.0" [features] -default = ["libsolv"] -libsolv = ["libsolv-sys", "libc"] -libsolv-rs = ["libsolv_rs"] +default = ["libsolv-sys"] +libsolv-sys = ["dep:libsolv-sys", "libc"] diff --git a/crates/rattler_solve/src/lib.rs b/crates/rattler_solve/src/lib.rs index 8d7ce3056..09bffa3f0 100644 --- a/crates/rattler_solve/src/lib.rs +++ b/crates/rattler_solve/src/lib.rs @@ -3,15 +3,15 @@ //! `rattler_solve` is a crate that provides functionality to solve Conda environments. It currently //! exposes the functionality through the [`SolverBackend::solve`] function. -#[cfg(feature = "libsolv")] +#[cfg(feature = "libsolv-sys")] mod libsolv; -#[cfg(feature = "libsolv-rs")] +#[cfg(feature = "libsolv_rs")] mod libsolv_rs; mod solver_backend; -#[cfg(feature = "libsolv-rs")] +#[cfg(feature = "libsolv_rs")] pub use crate::libsolv_rs::{LibsolvRsBackend, LibsolvRsRepoData}; -#[cfg(feature = "libsolv")] +#[cfg(feature = "libsolv-sys")] pub use libsolv::{ cache_repodata as cache_libsolv_repodata, LibcByteSlice, LibsolvBackend, LibsolvRepoData, }; @@ -80,546 +80,3 @@ pub struct SolverTask { /// The specs we want to solve pub specs: Vec, } - -#[cfg(test)] -mod test_libsolv { - use crate::libsolv::LibsolvBackend; - use crate::{LibsolvRepoData, LibsolvRsBackend, SolveError, SolverBackend, SolverTask}; - use rattler_conda_types::{ - Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, NoArchType, PackageRecord, - RepoData, RepoDataRecord, Version, - }; - use rattler_repodata_gateway::sparse::SparseRepoData; - use std::str::FromStr; - use url::Url; - - fn conda_json_path() -> String { - format!( - "{}/{}", - env!("CARGO_MANIFEST_DIR"), - "../../test-data/channels/conda-forge/linux-64/repodata.json" - ) - } - - fn conda_json_path_noarch() -> String { - format!( - "{}/{}", - env!("CARGO_MANIFEST_DIR"), - "../../test-data/channels/conda-forge/noarch/repodata.json" - ) - } - - fn dummy_channel_json_path() -> String { - format!( - "{}/{}", - env!("CARGO_MANIFEST_DIR"), - "../../test-data/channels/dummy/linux-64/repodata.json" - ) - } - - fn dummy_md5_hash() -> rattler_digest::Md5Hash { - rattler_digest::parse_digest_from_hex::( - "b3af409bb8423187c75e6c7f5b683908", - ) - .unwrap() - } - - fn dummy_sha256_hash() -> rattler_digest::Sha256Hash { - rattler_digest::parse_digest_from_hex::( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - ) - .unwrap() - } - - fn read_repodata(path: &str) -> Vec { - let repo_data: RepoData = - serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - repo_data.into_repo_data_records( - &Channel::from_str("conda-forge", &ChannelConfig::default()).unwrap(), - ) - } - - fn read_sparse_repodata(path: &str) -> SparseRepoData { - SparseRepoData::new( - Channel::from_str("dummy", &ChannelConfig::default()).unwrap(), - "dummy".to_string(), - path, - ) - .unwrap() - } - - fn installed_package( - channel: &str, - subdir: &str, - name: &str, - version: &str, - build: &str, - build_number: u64, - ) -> RepoDataRecord { - RepoDataRecord { - url: Url::from_str("http://example.com").unwrap(), - channel: channel.to_string(), - file_name: "dummy-filename".to_string(), - package_record: PackageRecord { - name: name.to_string(), - version: version.parse().unwrap(), - build: build.to_string(), - build_number, - subdir: subdir.to_string(), - md5: Some(dummy_md5_hash()), - sha256: Some(dummy_sha256_hash()), - size: None, - arch: None, - platform: None, - depends: Vec::new(), - constrains: Vec::new(), - track_features: Vec::new(), - features: None, - noarch: NoArchType::default(), - license: None, - license_family: None, - timestamp: None, - legacy_bz2_size: None, - legacy_bz2_md5: None, - }, - } - } - - fn solve_real_world(specs: Vec<&str>) -> Vec { - let specs = specs - .iter() - .map(|s| MatchSpec::from_str(s).unwrap()) - .collect::>(); - - let json_file = conda_json_path(); - let json_file_noarch = conda_json_path_noarch(); - - let sparse_repo_datas = vec![ - read_sparse_repodata(&json_file), - read_sparse_repodata(&json_file_noarch), - ]; - - let names = specs.iter().map(|s| s.name.clone().unwrap()); - let available_packages = - SparseRepoData::load_records_recursive(&sparse_repo_datas, names).unwrap(); - - let solver_task = SolverTask { - available_packages: &available_packages, - specs: specs.clone(), - locked_packages: Default::default(), - pinned_packages: Default::default(), - virtual_packages: Default::default(), - }; - - let pkgs1 = LibsolvBackend.solve(solver_task).unwrap(); - - let extract_pkgs = |records: Vec| { - let mut pkgs = records - .into_iter() - .map(|pkg| { - format!( - "{} {} {}", - pkg.package_record.name, - pkg.package_record.version, - pkg.package_record.build - ) - }) - .collect::>(); - - // The order of packages is nondeterministic, so we sort them to ensure we can compare them - // to a previous run - pkgs.sort(); - pkgs - }; - - extract_pkgs(pkgs1) - } - - #[cfg(feature = "libsolv")] - #[test] - fn test_solve_tensorboard_libsolv_sys() { - let libsolv_pkgs = - solve_real_world::(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); - insta::assert_yaml_snapshot!("solve_tensorboard_libsolv", libsolv_pkgs); - } - - #[cfg(feature = "libsolv")] - #[test] - fn test_solve_python_libsolv_sys() { - let libsolv_pkgs = solve_real_world::(vec!["python=3.9"]); - insta::assert_yaml_snapshot!("solve_python_libsolv", libsolv_pkgs); - } - - #[cfg(feature = "libsolv-rs")] - #[test] - fn test_solve_tensorboard_libsolv_rs() { - let libsolv_rs_pkgs = - solve_real_world::(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); - insta::assert_yaml_snapshot!("solve_tensorboard_libsolv_rs", libsolv_rs_pkgs); - } - - #[cfg(feature = "libsolv-rs")] - #[test] - fn test_solve_python_libsolv_rs() { - let libsolv_rs_pkgs = solve_real_world::(vec!["python=3.9"]); - insta::assert_yaml_snapshot!("solve_python_libsolv_rs", libsolv_rs_pkgs); - } - - #[test] - fn test_solve_dummy_repo_install_non_existent() { - let result = solve( - dummy_channel_json_path(), - Vec::new(), - Vec::new(), - &["asdfasdf", "foo<4"], - false, - ); - - assert!(result.is_err()); - - let err = result.err().unwrap(); - match err { - (SolveError::Unsolvable(libsolv_errors), SolveError::Unsolvable(libsolv_rs_errors)) => { - assert_eq!(libsolv_errors, vec!["nothing provides requested asdfasdf"]); - assert_eq!( - libsolv_rs_errors, - vec!["No candidates where found for asdfasdf.\n"] - ); - } - _ => panic!("Unexpected error: {err:?}"), - } - } - - #[test] - #[cfg(target_family = "unix")] - fn test_solve_with_cached_solv_file_install_new() { - let pkgs = solve( - dummy_channel_json_path(), - Vec::new(), - Vec::new(), - &["foo<4"], - true, - ) - .unwrap(); - - assert_eq!(1, pkgs.len()); - let info = &pkgs[0]; - - assert_eq!("foo-3.0.2-py36h1af98f8_1.conda", info.file_name); - assert_eq!( - "https://conda.anaconda.org/conda-forge/linux-64/foo-3.0.2-py36h1af98f8_1.conda", - info.url.to_string() - ); - assert_eq!("https://conda.anaconda.org/conda-forge/", info.channel); - assert_eq!("foo", info.package_record.name); - assert_eq!("linux-64", info.package_record.subdir); - assert_eq!("3.0.2", info.package_record.version.to_string()); - assert_eq!("py36h1af98f8_1", info.package_record.build); - assert_eq!(1, info.package_record.build_number); - assert_eq!( - rattler_digest::parse_digest_from_hex::( - "67a63bec3fd3205170eaad532d487595b8aaceb9814d13c6858d7bac3ef24cd4" - ) - .as_ref() - .unwrap(), - info.package_record.sha256.as_ref().unwrap() - ); - assert_eq!( - rattler_digest::parse_digest_from_hex::( - "fb731d9290f0bcbf3a054665f33ec94f" - ) - .as_ref() - .unwrap(), - info.package_record.md5.as_ref().unwrap() - ); - } - - #[test] - fn test_solve_dummy_repo_install_new() { - let pkgs = solve( - dummy_channel_json_path(), - Vec::new(), - Vec::new(), - &["foo<4"], - false, - ) - .unwrap(); - - assert_eq!(1, pkgs.len()); - let info = &pkgs[0]; - - assert_eq!("foo-3.0.2-py36h1af98f8_1.conda", info.file_name); - assert_eq!( - "https://conda.anaconda.org/conda-forge/linux-64/foo-3.0.2-py36h1af98f8_1.conda", - info.url.to_string() - ); - assert_eq!("https://conda.anaconda.org/conda-forge/", info.channel); - assert_eq!("foo", info.package_record.name); - assert_eq!("linux-64", info.package_record.subdir); - assert_eq!("3.0.2", info.package_record.version.to_string()); - assert_eq!("py36h1af98f8_1", info.package_record.build); - assert_eq!(1, info.package_record.build_number); - assert_eq!( - rattler_digest::parse_digest_from_hex::( - "67a63bec3fd3205170eaad532d487595b8aaceb9814d13c6858d7bac3ef24cd4" - ) - .as_ref() - .unwrap(), - info.package_record.sha256.as_ref().unwrap() - ); - assert_eq!( - rattler_digest::parse_digest_from_hex::( - "fb731d9290f0bcbf3a054665f33ec94f" - ) - .as_ref() - .unwrap(), - info.package_record.md5.as_ref().unwrap() - ); - } - - #[test] - fn test_solve_dummy_repo_prefers_conda_package() { - // There following package is provided as .tar.bz and as .conda in repodata.json - let match_spec = "foo=3.0.2=py36h1af98f8_1"; - - let operations = solve( - dummy_channel_json_path(), - Vec::new(), - Vec::new(), - &[match_spec], - false, - ) - .unwrap(); - - // The .conda entry is selected for installing - assert_eq!(operations.len(), 1); - assert_eq!(operations[0].file_name, "foo-3.0.2-py36h1af98f8_1.conda"); - } - - #[test] - fn test_solve_dummy_repo_install_noop() { - let already_installed = vec![installed_package( - "conda-forge", - "linux-64", - "foo", - "3.0.2", - "py36h1af98f8_1", - 1, - )]; - - let pkgs = solve( - dummy_channel_json_path(), - already_installed, - Vec::new(), - &["foo<4"], - false, - ) - .unwrap(); - - assert_eq!(1, pkgs.len()); - - // Install - let info = &pkgs[0]; - assert_eq!("foo", &info.package_record.name); - assert_eq!("3.0.2", &info.package_record.version.to_string()); - } - - #[test] - fn test_solve_dummy_repo_upgrade() { - let already_installed = vec![installed_package( - "conda-forge", - "linux-64", - "foo", - "3.0.2", - "py36h1af98f8_1", - 1, - )]; - - let pkgs = solve( - dummy_channel_json_path(), - already_installed, - Vec::new(), - &["foo>=4"], - false, - ) - .unwrap(); - - // Install - let info = &pkgs[0]; - assert_eq!("foo", &info.package_record.name); - assert_eq!("4.0.2", &info.package_record.version.to_string()); - } - - #[test] - fn test_solve_dummy_repo_downgrade() { - let already_installed = vec![installed_package( - "conda-forge", - "linux-64", - "foo", - "4.0.2", - "py36h1af98f8_1", - 1, - )]; - - let pkgs = solve( - dummy_channel_json_path(), - already_installed, - Vec::new(), - &["foo<4"], - false, - ) - .unwrap(); - - assert_eq!(pkgs.len(), 1); - - // Uninstall - let info = &pkgs[0]; - assert_eq!("foo", &info.package_record.name); - assert_eq!("3.0.2", &info.package_record.version.to_string()); - } - - #[test] - fn test_solve_dummy_repo_remove() { - let already_installed = vec![installed_package( - "conda-forge", - "linux-64", - "foo", - "3.0.2", - "py36h1af98f8_1", - 1, - )]; - - let pkgs = solve( - dummy_channel_json_path(), - already_installed, - Vec::new(), - &[], - false, - ) - .unwrap(); - - // Should be no packages! - assert_eq!(0, pkgs.len()); - } - - #[test] - fn test_solve_dummy_repo_with_virtual_package() { - let pkgs = solve( - dummy_channel_json_path(), - Vec::new(), - vec![GenericVirtualPackage { - name: "__unix".to_string(), - version: Version::from_str("0").unwrap(), - build_string: "0".to_string(), - }], - &["bar"], - false, - ) - .unwrap(); - - assert_eq!(pkgs.len(), 1); - - let info = &pkgs[0]; - assert_eq!("bar", &info.package_record.name); - assert_eq!("1.2.3", &info.package_record.version.to_string()); - } - - #[test] - fn test_solve_dummy_repo_missing_virtual_package() { - let result = solve( - dummy_channel_json_path(), - Vec::new(), - Vec::new(), - &["bar"], - false, - ); - - assert!(matches!( - result.err(), - Some((SolveError::Unsolvable(_), SolveError::Unsolvable(_))) - )); - } - - #[cfg(test)] - fn solve( - repo_path: String, - installed_packages: Vec, - virtual_packages: Vec, - match_specs: &[&str], - with_solv_file: bool, - ) -> Result, (SolveError, SolveError)> { - let repo_data = read_repodata(&repo_path); - - #[cfg(target_family = "unix")] - let cached_repo_data = super::cache_libsolv_repodata( - Channel::from_str("conda-forge", &ChannelConfig::default()) - .unwrap() - .platform_url(rattler_conda_types::Platform::Linux64) - .to_string(), - &repo_data, - ); - - let libsolv_repodata = if with_solv_file { - #[cfg(not(target_family = "unix"))] - panic!("solv files are unsupported for this platform"); - - #[cfg(target_family = "unix")] - LibsolvRepoData { - records: repo_data.iter().collect(), - solv_file: Some(&cached_repo_data), - } - } else { - LibsolvRepoData::from_iter(&repo_data) - }; - - let specs: Vec<_> = match_specs - .iter() - .map(|m| MatchSpec::from_str(m).unwrap()) - .collect(); - - let task = SolverTask { - locked_packages: installed_packages.clone(), - virtual_packages: virtual_packages.clone(), - available_packages: [libsolv_repodata], - specs: specs.clone(), - pinned_packages: Vec::new(), - }; - let pkgs_libsolv = LibsolvBackend.solve(task); - - let task = SolverTask { - locked_packages: installed_packages, - virtual_packages, - available_packages: [&repo_data], - specs, - pinned_packages: Vec::new(), - }; - let pkgs_libsolv_rs = LibsolvRsBackend.solve(task); - - if pkgs_libsolv.is_ok() != pkgs_libsolv_rs.is_ok() { - panic!("one of libsolv and libsolv_rs returns unsat, the other sat!"); - } - - if let Err(pkgs_libsolv) = pkgs_libsolv { - return Err((pkgs_libsolv, pkgs_libsolv_rs.err().unwrap())); - } - - let pkgs_libsolv = pkgs_libsolv.unwrap(); - let pkgs_libsolv_rs = pkgs_libsolv_rs.unwrap(); - if pkgs_libsolv != pkgs_libsolv_rs { - panic!("libsolv and libsolv_rs return different results!"); - } - - for pkg in pkgs_libsolv.iter() { - println!( - "{} {} {}", - pkg.package_record.name, pkg.package_record.version, pkg.package_record.build - ) - } - - if pkgs_libsolv.is_empty() { - println!("No packages in the environment!"); - } - - Ok(pkgs_libsolv) - } -} diff --git a/crates/rattler_solve/src/libsolv/mod.rs b/crates/rattler_solve/src/libsolv/mod.rs index 68ff97638..1ff232c58 100644 --- a/crates/rattler_solve/src/libsolv/mod.rs +++ b/crates/rattler_solve/src/libsolv/mod.rs @@ -74,6 +74,7 @@ fn c_string>(str: T) -> CString { } /// A [`SolverBackend`] implemented using the `libsolv` library +#[derive(Default)] pub struct LibsolvBackend; impl SolverBackend for LibsolvBackend { diff --git a/crates/rattler_solve/src/libsolv_rs/mod.rs b/crates/rattler_solve/src/libsolv_rs/mod.rs index 41befeec6..ca63dc965 100644 --- a/crates/rattler_solve/src/libsolv_rs/mod.rs +++ b/crates/rattler_solve/src/libsolv_rs/mod.rs @@ -28,6 +28,7 @@ impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRsRepoData<'a> { impl<'a> SolverRepoData<'a> for LibsolvRsRepoData<'a> {} /// A [`SolverBackend`] implemented using the `libsolv` library +#[derive(Default)] pub struct LibsolvRsBackend; impl SolverBackend for LibsolvRsBackend { diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs new file mode 100644 index 000000000..3cf0102cd --- /dev/null +++ b/crates/rattler_solve/tests/backends.rs @@ -0,0 +1,481 @@ +use rattler_conda_types::{ + Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, NoArchType, PackageRecord, RepoData, + RepoDataRecord, Version, +}; +use rattler_repodata_gateway::sparse::SparseRepoData; +use rattler_solve::{SolveError, SolverBackend, SolverTask}; +use std::str::FromStr; +use url::Url; + +fn conda_json_path() -> String { + format!( + "{}/{}", + env!("CARGO_MANIFEST_DIR"), + "../../test-data/channels/conda-forge/linux-64/repodata.json" + ) +} + +fn conda_json_path_noarch() -> String { + format!( + "{}/{}", + env!("CARGO_MANIFEST_DIR"), + "../../test-data/channels/conda-forge/noarch/repodata.json" + ) +} + +fn dummy_channel_json_path() -> String { + format!( + "{}/{}", + env!("CARGO_MANIFEST_DIR"), + "../../test-data/channels/dummy/linux-64/repodata.json" + ) +} + +fn dummy_md5_hash() -> rattler_digest::Md5Hash { + rattler_digest::parse_digest_from_hex::("b3af409bb8423187c75e6c7f5b683908") + .unwrap() +} + +fn dummy_sha256_hash() -> rattler_digest::Sha256Hash { + rattler_digest::parse_digest_from_hex::( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ) + .unwrap() +} + +fn read_repodata(path: &str) -> Vec { + let repo_data: RepoData = + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + repo_data.into_repo_data_records( + &Channel::from_str("conda-forge", &ChannelConfig::default()).unwrap(), + ) +} + +fn read_sparse_repodata(path: &str) -> SparseRepoData { + SparseRepoData::new( + Channel::from_str("dummy", &ChannelConfig::default()).unwrap(), + "dummy".to_string(), + path, + ) + .unwrap() +} + +fn installed_package( + channel: &str, + subdir: &str, + name: &str, + version: &str, + build: &str, + build_number: u64, +) -> RepoDataRecord { + RepoDataRecord { + url: Url::from_str("http://example.com").unwrap(), + channel: channel.to_string(), + file_name: "dummy-filename".to_string(), + package_record: PackageRecord { + name: name.to_string(), + version: version.parse().unwrap(), + build: build.to_string(), + build_number, + subdir: subdir.to_string(), + md5: Some(dummy_md5_hash()), + sha256: Some(dummy_sha256_hash()), + size: None, + arch: None, + platform: None, + depends: Vec::new(), + constrains: Vec::new(), + track_features: Vec::new(), + features: None, + noarch: NoArchType::default(), + license: None, + license_family: None, + timestamp: None, + legacy_bz2_size: None, + legacy_bz2_md5: None, + }, + } +} + +fn solve_real_world(specs: Vec<&str>) -> Vec { + let specs = specs + .iter() + .map(|s| MatchSpec::from_str(s).unwrap()) + .collect::>(); + + let json_file = conda_json_path(); + let json_file_noarch = conda_json_path_noarch(); + + let sparse_repo_datas = vec![ + read_sparse_repodata(&json_file), + read_sparse_repodata(&json_file_noarch), + ]; + + let names = specs.iter().map(|s| s.name.clone().unwrap()); + let available_packages = + SparseRepoData::load_records_recursive(&sparse_repo_datas, names).unwrap(); + + let solver_task = SolverTask { + available_packages: &available_packages, + specs: specs.clone(), + locked_packages: Default::default(), + pinned_packages: Default::default(), + virtual_packages: Default::default(), + }; + + let pkgs1 = T::default().solve(solver_task).unwrap(); + + let extract_pkgs = |records: Vec| { + let mut pkgs = records + .into_iter() + .map(|pkg| { + format!( + "{} {} {}", + pkg.package_record.name, pkg.package_record.version, pkg.package_record.build + ) + }) + .collect::>(); + + // The order of packages is nondeterministic, so we sort them to ensure we can compare them + // to a previous run + pkgs.sort(); + pkgs + }; + + extract_pkgs(pkgs1) +} + +macro_rules! solver_backend_tests { + ($T:ident) => { + #[test] + fn test_solve_tensorboard_libsolv_rs() { + insta::assert_yaml_snapshot!(solve_real_world::<$T>(vec![ + "tensorboard=2.1.1", + "grpc-cpp=1.39.1" + ])); + } + + #[test] + fn test_solve_python_libsolv_rs() { + insta::assert_yaml_snapshot!(solve_real_world::<$T>(vec!["python=3.9"])); + } + + #[test] + fn test_solve_dummy_repo_install_non_existent() { + let result = solve::<$T>( + dummy_channel_json_path(), + Vec::new(), + Vec::new(), + &["asdfasdf", "foo<4"], + ); + + assert!(result.is_err()); + + let err = result.err().unwrap(); + insta::assert_debug_snapshot!(err); + } + + #[test] + fn test_solve_dummy_repo_missing_virtual_package() { + let result = solve::<$T>(dummy_channel_json_path(), Vec::new(), Vec::new(), &["bar"]); + + assert!(matches!(result.err(), Some(SolveError::Unsolvable(_)))); + } + + #[test] + fn test_solve_dummy_repo_with_virtual_package() { + let pkgs = solve::<$T>( + dummy_channel_json_path(), + Vec::new(), + vec![GenericVirtualPackage { + name: "__unix".to_string(), + version: Version::from_str("0").unwrap(), + build_string: "0".to_string(), + }], + &["bar"], + ) + .unwrap(); + + assert_eq!(pkgs.len(), 1); + + let info = &pkgs[0]; + assert_eq!("bar", &info.package_record.name); + assert_eq!("1.2.3", &info.package_record.version.to_string()); + } + + #[test] + fn test_solve_dummy_repo_install_new() { + let pkgs = solve::<$T>( + dummy_channel_json_path(), + Vec::new(), + Vec::new(), + &["foo<4"], + ) + .unwrap(); + + assert_eq!(1, pkgs.len()); + let info = &pkgs[0]; + + assert_eq!("foo-3.0.2-py36h1af98f8_1.conda", info.file_name); + assert_eq!( + "https://conda.anaconda.org/conda-forge/linux-64/foo-3.0.2-py36h1af98f8_1.conda", + info.url.to_string() + ); + assert_eq!("https://conda.anaconda.org/conda-forge/", info.channel); + assert_eq!("foo", info.package_record.name); + assert_eq!("linux-64", info.package_record.subdir); + assert_eq!("3.0.2", info.package_record.version.to_string()); + assert_eq!("py36h1af98f8_1", info.package_record.build); + assert_eq!(1, info.package_record.build_number); + assert_eq!( + rattler_digest::parse_digest_from_hex::( + "67a63bec3fd3205170eaad532d487595b8aaceb9814d13c6858d7bac3ef24cd4" + ) + .as_ref() + .unwrap(), + info.package_record.sha256.as_ref().unwrap() + ); + assert_eq!( + rattler_digest::parse_digest_from_hex::( + "fb731d9290f0bcbf3a054665f33ec94f" + ) + .as_ref() + .unwrap(), + info.package_record.md5.as_ref().unwrap() + ); + } + + #[test] + fn test_solve_dummy_repo_prefers_conda_package() { + // There following package is provided as .tar.bz and as .conda in repodata.json + let match_spec = "foo=3.0.2=py36h1af98f8_1"; + + let operations = solve::<$T>( + dummy_channel_json_path(), + Vec::new(), + Vec::new(), + &[match_spec], + ) + .unwrap(); + + // The .conda entry is selected for installing + assert_eq!(operations.len(), 1); + assert_eq!(operations[0].file_name, "foo-3.0.2-py36h1af98f8_1.conda"); + } + + #[test] + fn test_solve_dummy_repo_install_noop() { + let already_installed = vec![installed_package( + "conda-forge", + "linux-64", + "foo", + "3.0.2", + "py36h1af98f8_1", + 1, + )]; + + let pkgs = solve::<$T>( + dummy_channel_json_path(), + already_installed, + Vec::new(), + &["foo<4"], + ) + .unwrap(); + + assert_eq!(1, pkgs.len()); + + // Install + let info = &pkgs[0]; + assert_eq!("foo", &info.package_record.name); + assert_eq!("3.0.2", &info.package_record.version.to_string()); + } + + #[test] + fn test_solve_dummy_repo_upgrade() { + let already_installed = vec![installed_package( + "conda-forge", + "linux-64", + "foo", + "3.0.2", + "py36h1af98f8_1", + 1, + )]; + + let pkgs = solve::<$T>( + dummy_channel_json_path(), + already_installed, + Vec::new(), + &["foo>=4"], + ) + .unwrap(); + + // Install + let info = &pkgs[0]; + assert_eq!("foo", &info.package_record.name); + assert_eq!("4.0.2", &info.package_record.version.to_string()); + } + + #[test] + fn test_solve_dummy_repo_downgrade() { + let already_installed = vec![installed_package( + "conda-forge", + "linux-64", + "foo", + "4.0.2", + "py36h1af98f8_1", + 1, + )]; + + let pkgs = solve::<$T>( + dummy_channel_json_path(), + already_installed, + Vec::new(), + &["foo<4"], + ) + .unwrap(); + + assert_eq!(pkgs.len(), 1); + + // Uninstall + let info = &pkgs[0]; + assert_eq!("foo", &info.package_record.name); + assert_eq!("3.0.2", &info.package_record.version.to_string()); + } + + #[test] + fn test_solve_dummy_repo_remove() { + let already_installed = vec![installed_package( + "conda-forge", + "linux-64", + "foo", + "3.0.2", + "py36h1af98f8_1", + 1, + )]; + + let pkgs = solve::<$T>( + dummy_channel_json_path(), + already_installed, + Vec::new(), + &[], + ) + .unwrap(); + + // Should be no packages! + assert_eq!(0, pkgs.len()); + } + }; +} + +#[cfg(feature = "libsolv-sys")] +mod libsolv_sys { + use super::*; + use rattler_solve::LibsolvBackend; + + solver_backend_tests!(LibsolvBackend); + + #[test] + #[cfg(target_family = "unix")] + fn test_solve_with_cached_solv_file_install_new() { + let repo_data = read_repodata(&repo_path); + + let cached_repo_data = super::cache_libsolv_repodata( + Channel::from_str("conda-forge", &ChannelConfig::default()) + .unwrap() + .platform_url(rattler_conda_types::Platform::Linux64) + .to_string(), + &repo_data, + ); + + let libsolv_repodata = LibsolvRepoData { + records: repo_data.iter().collect(), + solv_file: Some(&cached_repo_data), + }; + + let specs: Vec<_> = match_specs + .iter() + .map(|m| MatchSpec::from_str(m).unwrap()) + .collect(); + + let task = SolverTask { + locked_packages: installed_packages.clone(), + virtual_packages: virtual_packages.clone(), + available_packages: [&repo_data], + specs: specs.clone(), + pinned_packages: Vec::new(), + }; + + if pkgs.is_empty() { + println!("No packages in the environment!"); + } + + assert_eq!(1, pkgs.len()); + let info = &pkgs[0]; + + assert_eq!("foo-3.0.2-py36h1af98f8_1.conda", info.file_name); + assert_eq!( + "https://conda.anaconda.org/conda-forge/linux-64/foo-3.0.2-py36h1af98f8_1.conda", + info.url.to_string() + ); + assert_eq!("https://conda.anaconda.org/conda-forge/", info.channel); + assert_eq!("foo", info.package_record.name); + assert_eq!("linux-64", info.package_record.subdir); + assert_eq!("3.0.2", info.package_record.version.to_string()); + assert_eq!("py36h1af98f8_1", info.package_record.build); + assert_eq!(1, info.package_record.build_number); + assert_eq!( + rattler_digest::parse_digest_from_hex::( + "67a63bec3fd3205170eaad532d487595b8aaceb9814d13c6858d7bac3ef24cd4" + ) + .as_ref() + .unwrap(), + info.package_record.sha256.as_ref().unwrap() + ); + assert_eq!( + rattler_digest::parse_digest_from_hex::( + "fb731d9290f0bcbf3a054665f33ec94f" + ) + .as_ref() + .unwrap(), + info.package_record.md5.as_ref().unwrap() + ); + } +} + +#[cfg(feature = "libsolv_rs")] +mod libsolv_rs { + use super::*; + use rattler_solve::LibsolvRsBackend; + + solver_backend_tests!(LibsolvRsBackend); +} + +fn solve( + repo_path: String, + installed_packages: Vec, + virtual_packages: Vec, + match_specs: &[&str], +) -> Result, SolveError> { + let repo_data = read_repodata(&repo_path); + + let specs: Vec<_> = match_specs + .iter() + .map(|m| MatchSpec::from_str(m).unwrap()) + .collect(); + + let task = SolverTask { + locked_packages: installed_packages.clone(), + virtual_packages: virtual_packages.clone(), + available_packages: [&repo_data], + specs: specs.clone(), + pinned_packages: Vec::new(), + }; + + let pkgs = T::default().solve(task)?; + + if pkgs.is_empty() { + println!("No packages in the environment!"); + } + + Ok(pkgs) +} diff --git a/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_dummy_repo_install_non_existent.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_dummy_repo_install_non_existent.snap new file mode 100644 index 000000000..e5cee1cfa --- /dev/null +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_dummy_repo_install_non_existent.snap @@ -0,0 +1,10 @@ +--- +source: crates/rattler_solve/tests/backends.rs +assertion_line: 450 +expression: err +--- +Unsolvable( + [ + "No candidates where found for asdfasdf.\n", + ], +) diff --git a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv_rs.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_python_libsolv_rs.snap similarity index 80% rename from crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv_rs.snap rename to crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_python_libsolv_rs.snap index 4c0977ba1..a57cc1839 100644 --- a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv_rs.snap +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_python_libsolv_rs.snap @@ -1,6 +1,7 @@ --- -source: crates/rattler_solve/src/lib.rs -expression: libsolv_rs_pkgs +source: crates/rattler_solve/tests/backends.rs +assertion_line: 450 +expression: "solve_real_world::(vec![\"python=3.9\"])" --- - _libgcc_mutex 0.1 conda_forge - _openmp_mutex 4.5 2_gnu diff --git a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv_rs.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_tensorboard_libsolv_rs.snap similarity index 93% rename from crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv_rs.snap rename to crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_tensorboard_libsolv_rs.snap index 9618daa12..50bf1830b 100644 --- a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv_rs.snap +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_rs__solve_tensorboard_libsolv_rs.snap @@ -1,6 +1,7 @@ --- -source: crates/rattler_solve/src/lib.rs -expression: libsolv_rs_pkgs +source: crates/rattler_solve/tests/backends.rs +assertion_line: 450 +expression: "solve_real_world::(vec![\"tensorboard=2.1.1\",\n \"grpc-cpp=1.39.1\"])" --- - _libgcc_mutex 0.1 conda_forge - _openmp_mutex 4.5 2_gnu diff --git a/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_dummy_repo_install_non_existent.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_dummy_repo_install_non_existent.snap new file mode 100644 index 000000000..74a0b04a6 --- /dev/null +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_dummy_repo_install_non_existent.snap @@ -0,0 +1,10 @@ +--- +source: crates/rattler_solve/tests/backends.rs +assertion_line: 375 +expression: err +--- +Unsolvable( + [ + "nothing provides requested asdfasdf", + ], +) diff --git a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_python_libsolv_rs.snap similarity index 80% rename from crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv.snap rename to crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_python_libsolv_rs.snap index 5847bf573..3a6f3d66c 100644 --- a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_python_libsolv.snap +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_python_libsolv_rs.snap @@ -1,6 +1,7 @@ --- -source: crates/rattler_solve/src/lib.rs -expression: libsolv_pkgs +source: crates/rattler_solve/tests/backends.rs +assertion_line: 375 +expression: "solve_real_world::(vec![\"python=3.9\"])" --- - _libgcc_mutex 0.1 conda_forge - _openmp_mutex 4.5 2_gnu diff --git a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv.snap b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_tensorboard_libsolv_rs.snap similarity index 93% rename from crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv.snap rename to crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_tensorboard_libsolv_rs.snap index 5201e76a2..502c49d17 100644 --- a/crates/rattler_solve/src/snapshots/rattler_solve__test_libsolv__solve_tensorboard_libsolv.snap +++ b/crates/rattler_solve/tests/snapshots/backends__libsolv_sys__solve_tensorboard_libsolv_rs.snap @@ -1,6 +1,7 @@ --- -source: crates/rattler_solve/src/lib.rs -expression: libsolv_pkgs +source: crates/rattler_solve/tests/backends.rs +assertion_line: 375 +expression: "solve_real_world::(vec![\"tensorboard=2.1.1\",\n \"grpc-cpp=1.39.1\"])" --- - _libgcc_mutex 0.1 conda_forge - _openmp_mutex 4.5 2_gnu From d036b4131730ba86b31619ecf1e1bb315c9d99cc Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 4 Jul 2023 18:16:00 +0200 Subject: [PATCH 06/10] feat: added benchmarks --- crates/rattler_solve/Cargo.toml | 7 ++ crates/rattler_solve/benches/bench.rs | 92 ++++++++++++++++++++ crates/rattler_solve/tests/backends.rs | 116 +++++++++++++++++++++++-- 3 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 crates/rattler_solve/benches/bench.rs diff --git a/crates/rattler_solve/Cargo.toml b/crates/rattler_solve/Cargo.toml index 973555b69..a071fef71 100644 --- a/crates/rattler_solve/Cargo.toml +++ b/crates/rattler_solve/Cargo.toml @@ -30,7 +30,14 @@ insta = { version = "1.29.0", features = ["yaml"] } rstest = "0.17.0" serde_json = "1.0.96" url = "2.4.0" +similar-asserts = "1.4.2" +once_cell = "1.18.0" +criterion = "0.5.1" [features] default = ["libsolv-sys"] libsolv-sys = ["dep:libsolv-sys", "libc"] + +[[bench]] +name = "bench" +harness = false diff --git a/crates/rattler_solve/benches/bench.rs b/crates/rattler_solve/benches/bench.rs new file mode 100644 index 000000000..2bc578341 --- /dev/null +++ b/crates/rattler_solve/benches/bench.rs @@ -0,0 +1,92 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use rattler_conda_types::{Channel, ChannelConfig, MatchSpec}; +use rattler_repodata_gateway::sparse::SparseRepoData; +use rattler_solve::{SolverBackend, SolverTask}; +use std::str::FromStr; + +fn conda_json_path() -> String { + format!( + "{}/{}", + env!("CARGO_MANIFEST_DIR"), + "../../test-data/channels/conda-forge/linux-64/repodata.json" + ) +} + +fn conda_json_path_noarch() -> String { + format!( + "{}/{}", + env!("CARGO_MANIFEST_DIR"), + "../../test-data/channels/conda-forge/noarch/repodata.json" + ) +} + +fn read_sparse_repodata(path: &str) -> SparseRepoData { + SparseRepoData::new( + Channel::from_str("dummy", &ChannelConfig::default()).unwrap(), + "dummy".to_string(), + path, + ) + .unwrap() +} + +fn bench_solve_environment(c: &mut Criterion, specs: Vec<&str>) { + let name = specs.join(", "); + let mut group = c.benchmark_group(format!("solve {name}")); + + let specs = specs + .iter() + .map(|s| MatchSpec::from_str(s).unwrap()) + .collect::>(); + + let json_file = conda_json_path(); + let json_file_noarch = conda_json_path_noarch(); + + let sparse_repo_datas = vec![ + read_sparse_repodata(&json_file), + read_sparse_repodata(&json_file_noarch), + ]; + + let names = specs.iter().map(|s| s.name.clone().unwrap()); + let available_packages = + SparseRepoData::load_records_recursive(&sparse_repo_datas, names).unwrap(); + + #[cfg(feature = "libsolv-sys")] + group.bench_function("libsolv-sys", |b| { + b.iter(|| { + rattler_solve::LibsolvBackend + .solve(black_box(SolverTask { + available_packages: &available_packages, + locked_packages: vec![], + pinned_packages: vec![], + virtual_packages: vec![], + specs: specs.clone(), + })) + .unwrap() + }) + }); + + #[cfg(feature = "libsolv_rs")] + group.bench_function("libsolv_rs", |b| { + b.iter(|| { + rattler_solve::LibsolvRsBackend + .solve(black_box(SolverTask { + available_packages: &available_packages, + locked_packages: vec![], + pinned_packages: vec![], + virtual_packages: vec![], + specs: specs.clone(), + })) + .unwrap() + }) + }); + + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + bench_solve_environment(c, vec!["python=3.9"]); + bench_solve_environment(c, vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs index 3cf0102cd..81aab67d3 100644 --- a/crates/rattler_solve/tests/backends.rs +++ b/crates/rattler_solve/tests/backends.rs @@ -1,3 +1,4 @@ +use once_cell::sync::Lazy; use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, NoArchType, PackageRecord, RepoData, RepoDataRecord, Version, @@ -103,17 +104,11 @@ fn solve_real_world(specs: Vec<&str>) -> Vec .map(|s| MatchSpec::from_str(s).unwrap()) .collect::>(); - let json_file = conda_json_path(); - let json_file_noarch = conda_json_path_noarch(); - - let sparse_repo_datas = vec![ - read_sparse_repodata(&json_file), - read_sparse_repodata(&json_file_noarch), - ]; + let sparse_repo_datas = read_real_world_repo_data(); let names = specs.iter().map(|s| s.name.clone().unwrap()); let available_packages = - SparseRepoData::load_records_recursive(&sparse_repo_datas, names).unwrap(); + SparseRepoData::load_records_recursive(sparse_repo_datas, names).unwrap(); let solver_task = SolverTask { available_packages: &available_packages, @@ -145,6 +140,20 @@ fn solve_real_world(specs: Vec<&str>) -> Vec extract_pkgs(pkgs1) } +fn read_real_world_repo_data() -> &'static Vec { + static REPO_DATA: Lazy> = Lazy::new(|| { + let json_file = conda_json_path(); + let json_file_noarch = conda_json_path_noarch(); + + vec![ + read_sparse_repodata(&json_file), + read_sparse_repodata(&json_file_noarch), + ] + }); + + &REPO_DATA +} + macro_rules! solver_backend_tests { ($T:ident) => { #[test] @@ -479,3 +488,94 @@ fn solve( Ok(pkgs) } + +fn compare_solve(specs: Vec<&str>) { + let specs = specs + .iter() + .map(|s| MatchSpec::from_str(s).unwrap()) + .collect::>(); + + let sparse_repo_datas = read_real_world_repo_data(); + + let names = specs.iter().filter_map(|s| s.name.clone()); + let available_packages = + SparseRepoData::load_records_recursive(sparse_repo_datas, names).unwrap(); + + let extract_pkgs = |records: Vec| { + let mut pkgs = records + .into_iter() + .map(|pkg| { + format!( + "{} {} {}", + pkg.package_record.name, pkg.package_record.version, pkg.package_record.build + ) + }) + .collect::>(); + + // The order of packages is nondeterministic, so we sort them to ensure we can compare them + // to a previous run + pkgs.sort(); + pkgs + }; + + let mut results = Vec::new(); + + #[cfg(feature = "libsolv-sys")] + results.push(( + "libsolv-sys", + extract_pkgs( + rattler_solve::LibsolvBackend + .solve(SolverTask { + available_packages: &available_packages, + specs: specs.clone(), + locked_packages: Default::default(), + pinned_packages: Default::default(), + virtual_packages: Default::default(), + }) + .unwrap(), + ), + )); + + #[cfg(feature = "libsolv_rs")] + results.push(( + "libsolv_rs", + extract_pkgs( + rattler_solve::LibsolvRsBackend + .solve(SolverTask { + available_packages: &available_packages, + specs: specs.clone(), + locked_packages: Default::default(), + pinned_packages: Default::default(), + virtual_packages: Default::default(), + }) + .unwrap(), + ), + )); + + results.into_iter().fold(None, |previous, current| { + let previous = match previous { + Some(previous) => previous, + None => return Some(current), + }; + + similar_asserts::assert_eq!( + &previous.1, + ¤t.1, + "The result between {} and {} differs", + &previous.0, + ¤t.0 + ); + + Some(current) + }); +} + +#[test] +fn compare_solve_tensorboard() { + compare_solve(vec!["tensorboard=2.1.1", "grpc-cpp=1.39.1"]); +} + +#[test] +fn compare_solve_python() { + compare_solve(vec!["python=3.9"]); +} From 447b33b9404ff5c602889f527d4c67107274fe88 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Wed, 5 Jul 2023 16:58:37 +0200 Subject: [PATCH 07/10] fix: ci --- .github/workflows/rust-compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 1deacaf0e..f7a4d8002 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -14,7 +14,7 @@ env: RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always - DEFAULT_FEATURES: tokio,serde,reqwest,sparse,sysinfo,libsolv-rs + DEFAULT_FEATURES: tokio,serde,reqwest,sparse,sysinfo,libsolv_rs jobs: check: From b04ce8a655b355b471e002f11ff17012cadeafb5 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Wed, 5 Jul 2023 17:33:11 +0200 Subject: [PATCH 08/10] refactor: move implementations to modules --- crates/rattler-bin/src/commands/create.rs | 6 +- crates/rattler_solve/benches/bench.rs | 6 +- crates/rattler_solve/src/lib.rs | 74 ++++++++++++---- crates/rattler_solve/src/libsolv_rs/mod.rs | 24 +++--- .../src/{libsolv => libsolv_sys}/input.rs | 11 +-- .../libc_byte_slice.rs | 84 +++++++++---------- .../src/{libsolv => libsolv_sys}/mod.rs | 24 +++--- .../src/{libsolv => libsolv_sys}/output.rs | 12 +-- .../{libsolv => libsolv_sys}/wrapper/flags.rs | 0 .../{libsolv => libsolv_sys}/wrapper/keys.rs | 32 +++---- .../{libsolv => libsolv_sys}/wrapper/mod.rs | 0 .../{libsolv => libsolv_sys}/wrapper/pool.rs | 10 ++- .../{libsolv => libsolv_sys}/wrapper/queue.rs | 3 +- .../{libsolv => libsolv_sys}/wrapper/repo.rs | 9 +- .../wrapper/repodata.rs | 9 +- .../wrapper/solvable.rs | 6 +- .../wrapper/solve_goal.rs | 14 ++-- .../wrapper/solve_problem.rs | 0 .../wrapper/solver.rs | 19 ++--- .../wrapper/transaction.rs | 8 +- crates/rattler_solve/src/solver_backend.rs | 54 ------------ crates/rattler_solve/tests/backends.rs | 22 +++-- 22 files changed, 202 insertions(+), 225 deletions(-) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/input.rs (97%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/libc_byte_slice.rs (96%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/mod.rs (92%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/output.rs (90%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/flags.rs (100%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/keys.rs (98%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/mod.rs (100%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/pool.rs (98%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/queue.rs (97%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/repo.rs (96%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/repodata.rs (93%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/solvable.rs (95%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/solve_goal.rs (91%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/solve_problem.rs (100%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/solver.rs (93%) rename crates/rattler_solve/src/{libsolv => libsolv_sys}/wrapper/transaction.rs (93%) delete mode 100644 crates/rattler_solve/src/solver_backend.rs diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 0c5b86b52..2afa7f118 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -15,7 +15,7 @@ use rattler_repodata_gateway::fetch::{ CacheResult, DownloadProgress, FetchRepoDataError, FetchRepoDataOptions, }; use rattler_repodata_gateway::sparse::SparseRepoData; -use rattler_solve::{SolverBackend, SolverTask}; +use rattler_solve::{libsolv_rs, libsolv_sys, SolverImpl, SolverTask}; use reqwest::{Client, StatusCode}; use std::{ borrow::Cow, @@ -214,9 +214,9 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { let use_libsolv_rs = opt.use_experimental_libsolv_rs; let required_packages = wrap_in_progress("solving", move || { if use_libsolv_rs { - rattler_solve::LibsolvRsBackend.solve(solver_task) + libsolv_rs::Solver.solve(solver_task) } else { - rattler_solve::LibsolvBackend.solve(solver_task) + libsolv_sys::Solver.solve(solver_task) } })?; diff --git a/crates/rattler_solve/benches/bench.rs b/crates/rattler_solve/benches/bench.rs index 2bc578341..a0efe4bae 100644 --- a/crates/rattler_solve/benches/bench.rs +++ b/crates/rattler_solve/benches/bench.rs @@ -1,7 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rattler_conda_types::{Channel, ChannelConfig, MatchSpec}; use rattler_repodata_gateway::sparse::SparseRepoData; -use rattler_solve::{SolverBackend, SolverTask}; +use rattler_solve::{SolverImpl, SolverTask}; use std::str::FromStr; fn conda_json_path() -> String { @@ -53,7 +53,7 @@ fn bench_solve_environment(c: &mut Criterion, specs: Vec<&str>) { #[cfg(feature = "libsolv-sys")] group.bench_function("libsolv-sys", |b| { b.iter(|| { - rattler_solve::LibsolvBackend + rattler_solve::libsolv_sys::Solver .solve(black_box(SolverTask { available_packages: &available_packages, locked_packages: vec![], @@ -68,7 +68,7 @@ fn bench_solve_environment(c: &mut Criterion, specs: Vec<&str>) { #[cfg(feature = "libsolv_rs")] group.bench_function("libsolv_rs", |b| { b.iter(|| { - rattler_solve::LibsolvRsBackend + rattler_solve::libsolv_rs::Solver .solve(black_box(SolverTask { available_packages: &available_packages, locked_packages: vec![], diff --git a/crates/rattler_solve/src/lib.rs b/crates/rattler_solve/src/lib.rs index 09bffa3f0..160eae64f 100644 --- a/crates/rattler_solve/src/lib.rs +++ b/crates/rattler_solve/src/lib.rs @@ -1,25 +1,32 @@ -#![deny(missing_docs)] - //! `rattler_solve` is a crate that provides functionality to solve Conda environments. It currently -//! exposes the functionality through the [`SolverBackend::solve`] function. +//! exposes the functionality through the [`SolverImpl::solve`] function. -#[cfg(feature = "libsolv-sys")] -mod libsolv; -#[cfg(feature = "libsolv_rs")] -mod libsolv_rs; -mod solver_backend; +#![deny(missing_docs)] #[cfg(feature = "libsolv_rs")] -pub use crate::libsolv_rs::{LibsolvRsBackend, LibsolvRsRepoData}; +pub mod libsolv_rs; #[cfg(feature = "libsolv-sys")] -pub use libsolv::{ - cache_repodata as cache_libsolv_repodata, LibcByteSlice, LibsolvBackend, LibsolvRepoData, -}; -pub use solver_backend::SolverBackend; +pub mod libsolv_sys; + +use rattler_conda_types::{GenericVirtualPackage, MatchSpec, RepoDataRecord}; use std::fmt; -use rattler_conda_types::GenericVirtualPackage; -use rattler_conda_types::{MatchSpec, RepoDataRecord}; +/// Represents a solver implementation, capable of solving [`SolverTask`]s +pub trait SolverImpl { + /// The repo data associated to a channel and platform combination + type RepoData<'a>: SolverRepoData<'a>; + + /// Resolve the dependencies and return the [`RepoDataRecord`]s that should be present in the + /// environment. + fn solve< + 'a, + R: IntoRepoData<'a, Self::RepoData<'a>>, + TAvailablePackagesIterator: IntoIterator, + >( + &mut self, + task: SolverTask, + ) -> Result, SolveError>; +} /// Represents an error when solving the dependencies for a given environment #[derive(thiserror::Error, Debug)] @@ -80,3 +87,40 @@ pub struct SolverTask { /// The specs we want to solve pub specs: Vec, } + +/// A representation of a collection of [`RepoDataRecord`] usable by a [`SolverImpl`] +/// implementation. +/// +/// Some solvers might be able to cache the collection between different runs of the solver which +/// could potentially eliminate some overhead. This trait enables creating a representation of the +/// repodata that is most suitable for a specific backend. +/// +/// Some solvers may add additional functionality to their specific implementation that enables +/// caching the repodata to disk in an efficient way (see [`crate::libsolv::LibSolvRepoData`] for +/// an example). +pub trait SolverRepoData<'a>: FromIterator<&'a RepoDataRecord> {} + +/// Defines the ability to convert a type into [`SolverRepoData`]. +pub trait IntoRepoData<'a, S: SolverRepoData<'a>> { + /// Converts this instance into an instance of [`SolverRepoData`] which is consumable by a + /// specific [`SolverImpl`] implementation. + fn into(self) -> S; +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a Vec { + fn into(self) -> S { + S::from_iter(self.iter()) + } +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a [RepoDataRecord] { + fn into(self) -> S { + S::from_iter(self.iter()) + } +} + +impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for S { + fn into(self) -> S { + self + } +} diff --git a/crates/rattler_solve/src/libsolv_rs/mod.rs b/crates/rattler_solve/src/libsolv_rs/mod.rs index ca63dc965..935eae6c6 100644 --- a/crates/rattler_solve/src/libsolv_rs/mod.rs +++ b/crates/rattler_solve/src/libsolv_rs/mod.rs @@ -1,7 +1,9 @@ -use crate::solver_backend::{IntoRepoData, SolverRepoData}; -use crate::{SolveError, SolverBackend, SolverTask}; +//! Provides an solver implementation based on the [`libsolv_rs`] crate. + +use crate::{IntoRepoData, SolverRepoData}; +use crate::{SolveError, SolverTask}; use input::{add_repodata_records, add_virtual_packages}; -use libsolv_rs::{Pool, SolveJobs, Solver}; +use libsolv_rs::{Pool, SolveJobs, Solver as LibSolvRsSolver}; use output::get_required_packages; use rattler_conda_types::RepoDataRecord; use std::collections::HashMap; @@ -12,12 +14,12 @@ mod output; /// Represents the information required to load available packages into libsolv for a single channel /// and platform combination #[derive(Clone)] -pub struct LibsolvRsRepoData<'a> { +pub struct RepoData<'a> { /// The actual records after parsing `repodata.json` pub records: Vec<&'a RepoDataRecord>, } -impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRsRepoData<'a> { +impl<'a> FromIterator<&'a RepoDataRecord> for RepoData<'a> { fn from_iter>(iter: T) -> Self { Self { records: Vec::from_iter(iter), @@ -25,14 +27,14 @@ impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRsRepoData<'a> { } } -impl<'a> SolverRepoData<'a> for LibsolvRsRepoData<'a> {} +impl<'a> SolverRepoData<'a> for RepoData<'a> {} -/// A [`SolverBackend`] implemented using the `libsolv` library +/// A [`Solver`] implemented using the `libsolv` library #[derive(Default)] -pub struct LibsolvRsBackend; +pub struct Solver; -impl SolverBackend for LibsolvRsBackend { - type RepoData<'a> = LibsolvRsRepoData<'a>; +impl super::SolverImpl for Solver { + type RepoData<'a> = RepoData<'a>; fn solve< 'a, @@ -100,7 +102,7 @@ impl SolverBackend for LibsolvRsBackend { } // Construct a solver and solve the problems in the queue - let mut solver = Solver::new(pool); + let mut solver = LibSolvRsSolver::new(pool); let transaction = solver.solve(goal).map_err(|problem| { SolveError::Unsolvable(vec![problem.display_user_friendly(&solver).to_string()]) })?; diff --git a/crates/rattler_solve/src/libsolv/input.rs b/crates/rattler_solve/src/libsolv_sys/input.rs similarity index 97% rename from crates/rattler_solve/src/libsolv/input.rs rename to crates/rattler_solve/src/libsolv_sys/input.rs index 831f38e05..867028401 100644 --- a/crates/rattler_solve/src/libsolv/input.rs +++ b/crates/rattler_solve/src/libsolv_sys/input.rs @@ -1,13 +1,10 @@ //! Contains business logic that loads information into libsolv in order to solve a conda //! environment -use crate::libsolv::c_string; -use crate::libsolv::libc_byte_slice::LibcByteSlice; -use crate::libsolv::wrapper::keys::*; -use crate::libsolv::wrapper::pool::Pool; -use crate::libsolv::wrapper::repo::Repo; -use crate::libsolv::wrapper::repodata::Repodata; -use crate::libsolv::wrapper::solvable::SolvableId; +use super::{ + c_string, libc_byte_slice::LibcByteSlice, wrapper::keys::*, wrapper::pool::Pool, + wrapper::repo::Repo, wrapper::repodata::Repodata, wrapper::solvable::SolvableId, +}; use rattler_conda_types::package::ArchiveType; use rattler_conda_types::{GenericVirtualPackage, RepoDataRecord}; use std::cmp::Ordering; diff --git a/crates/rattler_solve/src/libsolv/libc_byte_slice.rs b/crates/rattler_solve/src/libsolv_sys/libc_byte_slice.rs similarity index 96% rename from crates/rattler_solve/src/libsolv/libc_byte_slice.rs rename to crates/rattler_solve/src/libsolv_sys/libc_byte_slice.rs index e4c9a8b38..4931c06a7 100644 --- a/crates/rattler_solve/src/libsolv/libc_byte_slice.rs +++ b/crates/rattler_solve/src/libsolv_sys/libc_byte_slice.rs @@ -1,42 +1,42 @@ -#![cfg_attr(not(target_family = "unix"), allow(dead_code))] - -use std::ptr::NonNull; - -/// Represents an owned byte slice that was allocated using [`libc::malloc`] and is deallocated upon -/// drop using [`libc::free`]. -pub struct LibcByteSlice { - ptr: NonNull, - len: usize, -} - -// We can safely implemenet `Send` because LibcByteSlice is immutable -unsafe impl Send for LibcByteSlice {} - -// We can safely implemenet `Send` because LibcByteSlice is immutable -unsafe impl Sync for LibcByteSlice {} - -impl LibcByteSlice { - /// Constructs a `LibcByteSlice` from its raw parts - /// - /// # Safety - /// - /// `ptr` should have been allocated using [`libc::malloc`] and `len` should be the size - /// in bytes of the allocated chunk of memory - pub unsafe fn from_raw_parts(ptr: NonNull, len: usize) -> LibcByteSlice { - LibcByteSlice { ptr, len } - } - - pub(super) fn as_ptr(&self) -> *mut libc::c_void { - self.ptr.as_ptr() - } - - pub(super) fn len(&self) -> usize { - self.len - } -} - -impl Drop for LibcByteSlice { - fn drop(&mut self) { - unsafe { libc::free(self.ptr.as_ptr() as *mut _) } - } -} +#![cfg_attr(not(target_family = "unix"), allow(dead_code))] + +use std::ptr::NonNull; + +/// Represents an owned byte slice that was allocated using [`libc::malloc`] and is deallocated upon +/// drop using [`libc::free`]. +pub struct LibcByteSlice { + ptr: NonNull, + len: usize, +} + +// We can safely implemenet `Send` because LibcByteSlice is immutable +unsafe impl Send for LibcByteSlice {} + +// We can safely implemenet `Send` because LibcByteSlice is immutable +unsafe impl Sync for LibcByteSlice {} + +impl LibcByteSlice { + /// Constructs a `LibcByteSlice` from its raw parts + /// + /// # Safety + /// + /// `ptr` should have been allocated using [`libc::malloc`] and `len` should be the size + /// in bytes of the allocated chunk of memory + pub unsafe fn from_raw_parts(ptr: NonNull, len: usize) -> LibcByteSlice { + LibcByteSlice { ptr, len } + } + + pub(super) fn as_ptr(&self) -> *mut libc::c_void { + self.ptr.as_ptr() + } + + pub(super) fn len(&self) -> usize { + self.len + } +} + +impl Drop for LibcByteSlice { + fn drop(&mut self) { + unsafe { libc::free(self.ptr.as_ptr() as *mut _) } + } +} diff --git a/crates/rattler_solve/src/libsolv/mod.rs b/crates/rattler_solve/src/libsolv_sys/mod.rs similarity index 92% rename from crates/rattler_solve/src/libsolv/mod.rs rename to crates/rattler_solve/src/libsolv_sys/mod.rs index 1ff232c58..301ba1fa7 100644 --- a/crates/rattler_solve/src/libsolv/mod.rs +++ b/crates/rattler_solve/src/libsolv_sys/mod.rs @@ -1,5 +1,7 @@ -use crate::solver_backend::{IntoRepoData, SolverRepoData}; -use crate::{SolveError, SolverBackend, SolverTask}; +//! Provides an solver implementation based on the [`libsolv-sys`] crate. + +use crate::{IntoRepoData, SolverRepoData}; +use crate::{SolveError, SolverTask}; pub use input::cache_repodata; use input::{add_repodata_records, add_solv_file, add_virtual_packages}; pub use libc_byte_slice::LibcByteSlice; @@ -22,7 +24,7 @@ mod wrapper; /// Represents the information required to load available packages into libsolv for a single channel /// and platform combination #[derive(Clone)] -pub struct LibsolvRepoData<'a> { +pub struct RepoData<'a> { /// The actual records after parsing `repodata.json` pub records: Vec<&'a RepoDataRecord>, @@ -30,7 +32,7 @@ pub struct LibsolvRepoData<'a> { pub solv_file: Option<&'a LibcByteSlice>, } -impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRepoData<'a> { +impl<'a> FromIterator<&'a RepoDataRecord> for RepoData<'a> { fn from_iter>(iter: T) -> Self { Self { records: Vec::from_iter(iter), @@ -39,7 +41,7 @@ impl<'a> FromIterator<&'a RepoDataRecord> for LibsolvRepoData<'a> { } } -impl<'a> LibsolvRepoData<'a> { +impl<'a> RepoData<'a> { /// Constructs a new `LibsolvRsRepoData` #[deprecated(since = "0.6.0", note = "use From::from instead")] pub fn from_records(records: impl Into>) -> Self { @@ -50,7 +52,7 @@ impl<'a> LibsolvRepoData<'a> { } } -impl<'a> SolverRepoData<'a> for LibsolvRepoData<'a> {} +impl<'a> SolverRepoData<'a> for RepoData<'a> {} /// Convenience method that converts a string reference to a CString, replacing NUL characters with /// whitespace (` `) @@ -73,12 +75,12 @@ fn c_string>(str: T) -> CString { unsafe { CString::from_vec_with_nul_unchecked(vec) } } -/// A [`SolverBackend`] implemented using the `libsolv` library +/// A [`Solver`] implemented using the `libsolv` library #[derive(Default)] -pub struct LibsolvBackend; +pub struct Solver; -impl SolverBackend for LibsolvBackend { - type RepoData<'a> = LibsolvRepoData<'a>; +impl super::SolverImpl for Solver { + type RepoData<'a> = RepoData<'a>; fn solve< 'a, @@ -195,7 +197,7 @@ impl SolverBackend for LibsolvBackend { #[cfg(test)] mod test { - use crate::libsolv::c_string; + use super::*; use rstest::rstest; #[rstest] diff --git a/crates/rattler_solve/src/libsolv/output.rs b/crates/rattler_solve/src/libsolv_sys/output.rs similarity index 90% rename from crates/rattler_solve/src/libsolv/output.rs rename to crates/rattler_solve/src/libsolv_sys/output.rs index 68424b983..58ff8f3a0 100644 --- a/crates/rattler_solve/src/libsolv/output.rs +++ b/crates/rattler_solve/src/libsolv_sys/output.rs @@ -1,11 +1,13 @@ //! Contains business logic to retrieve the results from libsolv after attempting to resolve a conda //! environment -use crate::libsolv::wrapper::pool::{Pool, StringId}; -use crate::libsolv::wrapper::repo::RepoId; -use crate::libsolv::wrapper::solvable::SolvableId; -use crate::libsolv::wrapper::transaction::Transaction; -use crate::libsolv::wrapper::{ffi, solvable}; +use super::{ + wrapper::pool::{Pool, StringId}, + wrapper::repo::RepoId, + wrapper::solvable::SolvableId, + wrapper::transaction::Transaction, + wrapper::{ffi, solvable}, +}; use rattler_conda_types::RepoDataRecord; use std::collections::HashMap; diff --git a/crates/rattler_solve/src/libsolv/wrapper/flags.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/flags.rs similarity index 100% rename from crates/rattler_solve/src/libsolv/wrapper/flags.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/flags.rs diff --git a/crates/rattler_solve/src/libsolv/wrapper/keys.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/keys.rs similarity index 98% rename from crates/rattler_solve/src/libsolv/wrapper/keys.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/keys.rs index 88650eab1..b820000a2 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/keys.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/keys.rs @@ -1,16 +1,16 @@ -// The constants below are taken from -// https://github.com/openSUSE/libsolv/blob/67aaf74844c532129ec8d7c8a7be4209ee4ef78d/src/knownid.h -// -// We have only copied those that are interesting for rattler - -pub const SOLVABLE_LICENSE: &str = "solvable:license"; -pub const SOLVABLE_BUILDTIME: &str = "solvable:buildtime"; -pub const SOLVABLE_DOWNLOADSIZE: &str = "solvable:downloadsize"; -pub const SOLVABLE_CHECKSUM: &str = "solvable:checksum"; -pub const SOLVABLE_PKGID: &str = "solvable:pkgid"; -pub const SOLVABLE_BUILDFLAVOR: &str = "solvable:buildflavor"; -pub const SOLVABLE_BUILDVERSION: &str = "solvable:buildversion"; -pub const REPOKEY_TYPE_MD5: &str = "repokey:type:md5"; -pub const REPOKEY_TYPE_SHA256: &str = "repokey:type:sha256"; -pub const SOLVABLE_CONSTRAINS: &str = "solvable:constrains"; -pub const SOLVABLE_TRACK_FEATURES: &str = "solvable:track_features"; +// The constants below are taken from +// https://github.com/openSUSE/libsolv/blob/67aaf74844c532129ec8d7c8a7be4209ee4ef78d/src/knownid.h +// +// We have only copied those that are interesting for rattler + +pub const SOLVABLE_LICENSE: &str = "solvable:license"; +pub const SOLVABLE_BUILDTIME: &str = "solvable:buildtime"; +pub const SOLVABLE_DOWNLOADSIZE: &str = "solvable:downloadsize"; +pub const SOLVABLE_CHECKSUM: &str = "solvable:checksum"; +pub const SOLVABLE_PKGID: &str = "solvable:pkgid"; +pub const SOLVABLE_BUILDFLAVOR: &str = "solvable:buildflavor"; +pub const SOLVABLE_BUILDVERSION: &str = "solvable:buildversion"; +pub const REPOKEY_TYPE_MD5: &str = "repokey:type:md5"; +pub const REPOKEY_TYPE_SHA256: &str = "repokey:type:sha256"; +pub const SOLVABLE_CONSTRAINS: &str = "solvable:constrains"; +pub const SOLVABLE_TRACK_FEATURES: &str = "solvable:track_features"; diff --git a/crates/rattler_solve/src/libsolv/wrapper/mod.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/mod.rs similarity index 100% rename from crates/rattler_solve/src/libsolv/wrapper/mod.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/mod.rs diff --git a/crates/rattler_solve/src/libsolv/wrapper/pool.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/pool.rs similarity index 98% rename from crates/rattler_solve/src/libsolv/wrapper/pool.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/pool.rs index 62e733c45..13ae809f7 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/pool.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/pool.rs @@ -1,6 +1,10 @@ -use super::{ffi, repo::Repo, solvable::SolvableId, solver::Solver}; -use crate::libsolv::c_string; -use crate::libsolv::wrapper::ffi::Id; +use super::{ + super::{c_string, wrapper::ffi::Id}, + ffi, + repo::Repo, + solvable::SolvableId, + solver::Solver, +}; use rattler_conda_types::MatchSpec; use std::ffi::c_char; use std::{ diff --git a/crates/rattler_solve/src/libsolv/wrapper/queue.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/queue.rs similarity index 97% rename from crates/rattler_solve/src/libsolv/wrapper/queue.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/queue.rs index fc5eacb67..6367f9ca8 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/queue.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/queue.rs @@ -1,5 +1,4 @@ -use super::ffi; -use crate::libsolv::wrapper::solvable::SolvableId; +use super::{ffi, solvable::SolvableId}; use std::marker::PhantomData; /// Wrapper for libsolv queue type. This type is used by to gather items of a specific type. This diff --git a/crates/rattler_solve/src/libsolv/wrapper/repo.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/repo.rs similarity index 96% rename from crates/rattler_solve/src/libsolv/wrapper/repo.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/repo.rs index 394bcf52c..a368a85e8 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/repo.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/repo.rs @@ -1,6 +1,4 @@ -use super::repodata::Repodata; -use super::{ffi, pool::Pool, solvable::SolvableId}; -use crate::libsolv::c_string; +use super::{super::c_string, ffi, pool::Pool, repodata::Repodata, solvable::SolvableId}; use std::{marker::PhantomData, ptr::NonNull}; /// Wrapper for libsolv repo, which contains package information (in our case, we are creating repos @@ -121,10 +119,7 @@ impl<'pool> Repo<'pool> { #[cfg(test)] mod tests { - use super::super::pool::Pool; - use super::Repo; - use crate::libsolv::c_string; - use crate::libsolv::wrapper::pool::StringId; + use super::{super::pool::StringId, *}; #[test] fn test_repo_creation() { diff --git a/crates/rattler_solve/src/libsolv/wrapper/repodata.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/repodata.rs similarity index 93% rename from crates/rattler_solve/src/libsolv/wrapper/repodata.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/repodata.rs index 62c434330..7ba1523b9 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/repodata.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/repodata.rs @@ -1,10 +1,5 @@ -use super::ffi; -use crate::libsolv::wrapper::pool::StringId; -use crate::libsolv::wrapper::repo::Repo; -use crate::libsolv::wrapper::solvable::SolvableId; -use std::ffi::CStr; -use std::marker::PhantomData; -use std::ptr::NonNull; +use super::{ffi, pool::StringId, repo::Repo, solvable::SolvableId}; +use std::{ffi::CStr, marker::PhantomData, ptr::NonNull}; /// Wrapper for libsolv repodata, which provides functions to manipulate solvables /// diff --git a/crates/rattler_solve/src/libsolv/wrapper/solvable.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/solvable.rs similarity index 95% rename from crates/rattler_solve/src/libsolv/wrapper/solvable.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/solvable.rs index d086cb4b1..43e4f12d2 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/solvable.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/solvable.rs @@ -1,5 +1,7 @@ -use super::ffi; -use super::pool::{Pool, StringId}; +use super::{ + ffi, + pool::{Pool, StringId}, +}; use std::ptr::NonNull; /// Represents a solvable in a [`Repo`] or [`Pool`] diff --git a/crates/rattler_solve/src/libsolv/wrapper/solve_goal.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/solve_goal.rs similarity index 91% rename from crates/rattler_solve/src/libsolv/wrapper/solve_goal.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/solve_goal.rs index a308f807a..cf2f2178f 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/solve_goal.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/solve_goal.rs @@ -1,10 +1,12 @@ -use super::ffi; -use crate::libsolv::wrapper::ffi::{ - SOLVER_DISFAVOR, SOLVER_ERASE, SOLVER_FAVOR, SOLVER_INSTALL, SOLVER_LOCK, SOLVER_SOLVABLE, - SOLVER_SOLVABLE_PROVIDES, SOLVER_UPDATE, SOLVER_WEAK, +use super::{ + ffi, + ffi::{ + SOLVER_DISFAVOR, SOLVER_ERASE, SOLVER_FAVOR, SOLVER_INSTALL, SOLVER_LOCK, SOLVER_SOLVABLE, + SOLVER_SOLVABLE_PROVIDES, SOLVER_UPDATE, SOLVER_WEAK, + }, + pool::MatchSpecId, + solvable::SolvableId, }; -use crate::libsolv::wrapper::pool::MatchSpecId; -use crate::libsolv::wrapper::solvable::SolvableId; use std::os::raw::c_int; /// Wrapper for libsolv queue type that stores jobs for the solver. This type provides a safe diff --git a/crates/rattler_solve/src/libsolv/wrapper/solve_problem.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/solve_problem.rs similarity index 100% rename from crates/rattler_solve/src/libsolv/wrapper/solve_problem.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/solve_problem.rs diff --git a/crates/rattler_solve/src/libsolv/wrapper/solver.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/solver.rs similarity index 93% rename from crates/rattler_solve/src/libsolv/wrapper/solver.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/solver.rs index 137d1f2a3..7240e947d 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/solver.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/solver.rs @@ -1,16 +1,9 @@ -use std::ffi::CStr; -use std::marker::PhantomData; -use std::ptr::NonNull; - -use crate::libsolv::wrapper::pool::Pool; -use crate::libsolv::wrapper::solve_goal::SolveGoal; -use crate::libsolv::wrapper::solve_problem::SolveProblem; - -use super::ffi; -use super::flags::SolverFlag; -use super::queue::Queue; -use super::solvable::SolvableId; -use super::transaction::Transaction; +use std::{ffi::CStr, marker::PhantomData, ptr::NonNull}; + +use super::{ + ffi, flags::SolverFlag, pool::Pool, queue::Queue, solvable::SolvableId, solve_goal::SolveGoal, + solve_problem::SolveProblem, transaction::Transaction, +}; /// Wrapper for libsolv solver, which is used to drive dependency resolution /// diff --git a/crates/rattler_solve/src/libsolv/wrapper/transaction.rs b/crates/rattler_solve/src/libsolv_sys/wrapper/transaction.rs similarity index 93% rename from crates/rattler_solve/src/libsolv/wrapper/transaction.rs rename to crates/rattler_solve/src/libsolv_sys/wrapper/transaction.rs index d083ab49a..9c589eb2a 100644 --- a/crates/rattler_solve/src/libsolv/wrapper/transaction.rs +++ b/crates/rattler_solve/src/libsolv_sys/wrapper/transaction.rs @@ -1,9 +1,5 @@ -use super::ffi; -use super::solvable::SolvableId; -use super::solver::Solver; -use crate::libsolv::wrapper::queue::QueueRef; -use std::marker::PhantomData; -use std::ptr::NonNull; +use super::{ffi, queue::QueueRef, solvable::SolvableId, solver::Solver}; +use std::{marker::PhantomData, ptr::NonNull}; /// Wrapper for [`ffi::Transaction`], which is an abstraction over changes that need to be /// done to satisfy the dependency constraints diff --git a/crates/rattler_solve/src/solver_backend.rs b/crates/rattler_solve/src/solver_backend.rs deleted file mode 100644 index d278c4168..000000000 --- a/crates/rattler_solve/src/solver_backend.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::{SolveError, SolverTask}; -use rattler_conda_types::RepoDataRecord; - -/// A representation of a collection of [`RepoDataRecord`] usable by a [`SolverBackend`] -/// implementation. -/// -/// Some solvers might be able to cache the collection between different runs of the solver which -/// could potentially eliminate some overhead. This trait enables creating a representation of the -/// repodata that is most suitable for a specific backend. -/// -/// Some solvers may add additional functionality to their specific implementation that enables -/// caching the repodata to disk in an efficient way (see [`crate::libsolv::LibSolvRepoData`] for -/// an example). -pub trait SolverRepoData<'a>: FromIterator<&'a RepoDataRecord> {} - -/// Defines the ability to convert a type into [`SolverRepoData`]. -pub trait IntoRepoData<'a, S: SolverRepoData<'a>> { - fn into(self) -> S; -} - -impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a Vec { - fn into(self) -> S { - S::from_iter(self.iter()) - } -} - -impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for &'a [RepoDataRecord] { - fn into(self) -> S { - S::from_iter(self.iter()) - } -} - -impl<'a, S: SolverRepoData<'a>> IntoRepoData<'a, S> for S { - fn into(self) -> S { - self - } -} - -/// Represents a solver backend, capable of solving [`SolverTask`]s -pub trait SolverBackend { - /// The repo data associated to a channel and platform combination - type RepoData<'a>: SolverRepoData<'a>; - - /// Resolve the dependencies and return the [`RepoDataRecord`]s that should be present in the - /// environment. - fn solve< - 'a, - R: IntoRepoData<'a, Self::RepoData<'a>>, - TAvailablePackagesIterator: IntoIterator, - >( - &mut self, - task: SolverTask, - ) -> Result, SolveError>; -} diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs index 81aab67d3..1e58252f0 100644 --- a/crates/rattler_solve/tests/backends.rs +++ b/crates/rattler_solve/tests/backends.rs @@ -4,7 +4,7 @@ use rattler_conda_types::{ RepoDataRecord, Version, }; use rattler_repodata_gateway::sparse::SparseRepoData; -use rattler_solve::{SolveError, SolverBackend, SolverTask}; +use rattler_solve::{SolveError, SolverImpl, SolverTask}; use std::str::FromStr; use url::Url; @@ -98,7 +98,7 @@ fn installed_package( } } -fn solve_real_world(specs: Vec<&str>) -> Vec { +fn solve_real_world(specs: Vec<&str>) -> Vec { let specs = specs .iter() .map(|s| MatchSpec::from_str(s).unwrap()) @@ -155,7 +155,7 @@ fn read_real_world_repo_data() -> &'static Vec { } macro_rules! solver_backend_tests { - ($T:ident) => { + ($T:path) => { #[test] fn test_solve_tensorboard_libsolv_rs() { insta::assert_yaml_snapshot!(solve_real_world::<$T>(vec![ @@ -379,16 +379,15 @@ macro_rules! solver_backend_tests { #[cfg(feature = "libsolv-sys")] mod libsolv_sys { use super::*; - use rattler_solve::LibsolvBackend; - solver_backend_tests!(LibsolvBackend); + solver_backend_tests!(rattler_solve::libsolv_sys::Solver); #[test] #[cfg(target_family = "unix")] fn test_solve_with_cached_solv_file_install_new() { let repo_data = read_repodata(&repo_path); - let cached_repo_data = super::cache_libsolv_repodata( + let cached_repo_data = rattler_solve::libsolv_sys::cache_repodata( Channel::from_str("conda-forge", &ChannelConfig::default()) .unwrap() .platform_url(rattler_conda_types::Platform::Linux64) @@ -396,7 +395,7 @@ mod libsolv_sys { &repo_data, ); - let libsolv_repodata = LibsolvRepoData { + let libsolv_repodata = rattler_solve::libsolv_sys::RepoData { records: repo_data.iter().collect(), solv_file: Some(&cached_repo_data), }; @@ -454,12 +453,11 @@ mod libsolv_sys { #[cfg(feature = "libsolv_rs")] mod libsolv_rs { use super::*; - use rattler_solve::LibsolvRsBackend; - solver_backend_tests!(LibsolvRsBackend); + solver_backend_tests!(rattler_solve::libsolv_rs::Solver); } -fn solve( +fn solve( repo_path: String, installed_packages: Vec, virtual_packages: Vec, @@ -524,7 +522,7 @@ fn compare_solve(specs: Vec<&str>) { results.push(( "libsolv-sys", extract_pkgs( - rattler_solve::LibsolvBackend + rattler_solve::libsolv_sys::Solver .solve(SolverTask { available_packages: &available_packages, specs: specs.clone(), @@ -540,7 +538,7 @@ fn compare_solve(specs: Vec<&str>) { results.push(( "libsolv_rs", extract_pkgs( - rattler_solve::LibsolvRsBackend + rattler_solve::libsolv_rs::Solver .solve(SolverTask { available_packages: &available_packages, specs: specs.clone(), From 939e3e63f4c9ab2eaecf92bbf1a85b78c84a43ac Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Wed, 5 Jul 2023 17:36:34 +0200 Subject: [PATCH 09/10] fix: docs --- crates/rattler_solve/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_solve/src/lib.rs b/crates/rattler_solve/src/lib.rs index 160eae64f..fcb784644 100644 --- a/crates/rattler_solve/src/lib.rs +++ b/crates/rattler_solve/src/lib.rs @@ -96,7 +96,7 @@ pub struct SolverTask { /// repodata that is most suitable for a specific backend. /// /// Some solvers may add additional functionality to their specific implementation that enables -/// caching the repodata to disk in an efficient way (see [`crate::libsolv::LibSolvRepoData`] for +/// caching the repodata to disk in an efficient way (see [`crate::libsolv_sys::RepoData`] for /// an example). pub trait SolverRepoData<'a>: FromIterator<&'a RepoDataRecord> {} From c1c34bfdbb73e631d9d4a16adf28baab2170e184 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Wed, 5 Jul 2023 21:06:04 +0200 Subject: [PATCH 10/10] fix: unix test --- crates/rattler_solve/tests/backends.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs index 1e58252f0..9d0c6e7de 100644 --- a/crates/rattler_solve/tests/backends.rs +++ b/crates/rattler_solve/tests/backends.rs @@ -385,7 +385,7 @@ mod libsolv_sys { #[test] #[cfg(target_family = "unix")] fn test_solve_with_cached_solv_file_install_new() { - let repo_data = read_repodata(&repo_path); + let repo_data = read_repodata(&dummy_channel_json_path()); let cached_repo_data = rattler_solve::libsolv_sys::cache_repodata( Channel::from_str("conda-forge", &ChannelConfig::default()) @@ -400,18 +400,17 @@ mod libsolv_sys { solv_file: Some(&cached_repo_data), }; - let specs: Vec<_> = match_specs - .iter() - .map(|m| MatchSpec::from_str(m).unwrap()) - .collect(); - - let task = SolverTask { - locked_packages: installed_packages.clone(), - virtual_packages: virtual_packages.clone(), - available_packages: [&repo_data], - specs: specs.clone(), - pinned_packages: Vec::new(), - }; + let specs: Vec = vec!["foo<4".parse().unwrap()]; + + let pkgs = rattler_solve::libsolv_sys::Solver + .solve(SolverTask { + locked_packages: Vec::new(), + virtual_packages: Vec::new(), + available_packages: [libsolv_repodata], + specs: specs.clone(), + pinned_packages: Vec::new(), + }) + .unwrap(); if pkgs.is_empty() { println!("No packages in the environment!");