diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index c1d3ac46f..1ed5b7ac6 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -20,6 +20,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + lfs: true - run: | curl -fsSL https://pixi.sh/install.sh | bash echo "/home/runner/.pixi/bin" >> $GITHUB_PATH diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 31f35a7f8..5492602a5 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -587,7 +587,7 @@ async fn fetch_repo_data_records_with_progress( let result = rattler_repodata_gateway::fetch::fetch_repo_data( channel.platform_url(platform), client, - repodata_cache, + repodata_cache.to_path_buf(), FetchRepoDataOptions::default(), Some(Box::new(move |DownloadProgress { total, bytes }| { download_progress_progress_bar.set_length(total.unwrap_or(bytes)); diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index 6e1f97caa..6ea622990 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -283,7 +283,7 @@ async fn repodata_from_file( pub async fn fetch_repo_data( subdir_url: Url, client: AuthenticatedClient, - cache_path: &Path, + cache_path: PathBuf, options: FetchRepoDataOptions, progress: Option, ) -> Result { @@ -321,7 +321,7 @@ pub async fn fetch_repo_data( // Validate the current state of the cache let cache_state = if cache_action != CacheAction::NoCache { let owned_subdir_url = subdir_url.clone(); - let owned_cache_path = cache_path.to_owned(); + let owned_cache_path = cache_path.clone(); let owned_cache_key = cache_key.clone(); let cache_state = tokio::task::spawn_blocking(move || { validate_cached_state(&owned_cache_path, &owned_subdir_url, &owned_cache_key) @@ -530,7 +530,7 @@ pub async fn fetch_repo_data( } else { Encoding::Passthrough }, - cache_path, + &cache_path, progress, ) .await?; @@ -1114,7 +1114,7 @@ mod test { let result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1144,7 +1144,7 @@ mod test { let CachedRepoData { cache_result, .. } = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.path().to_owned(), Default::default(), None, ) @@ -1157,7 +1157,7 @@ mod test { let CachedRepoData { cache_result, .. } = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.path().to_owned(), Default::default(), None, ) @@ -1181,7 +1181,7 @@ mod test { let CachedRepoData { cache_result, .. } = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1210,7 +1210,7 @@ mod test { let result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1252,7 +1252,7 @@ mod test { let result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1301,7 +1301,7 @@ mod test { let result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1354,7 +1354,7 @@ mod test { let result = fetch_repo_data( server.url(), authenticated_client, - cache_dir.path(), + cache_dir.into_path(), Default::default(), None, ) @@ -1387,7 +1387,7 @@ mod test { let _result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), FetchRepoDataOptions { ..Default::default() }, @@ -1412,7 +1412,7 @@ mod test { Url::parse(format!("file://{}", subdir_path.path().to_str().unwrap()).as_str()) .unwrap(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), FetchRepoDataOptions { ..Default::default() }, @@ -1436,7 +1436,7 @@ mod test { let result = fetch_repo_data( server.url(), AuthenticatedClient::default(), - cache_dir.path(), + cache_dir.into_path(), FetchRepoDataOptions { ..Default::default() }, diff --git a/crates/rattler_repodata_gateway/src/lib.rs b/crates/rattler_repodata_gateway/src/lib.rs index 86db18582..747001896 100644 --- a/crates/rattler_repodata_gateway/src/lib.rs +++ b/crates/rattler_repodata_gateway/src/lib.rs @@ -26,7 +26,7 @@ //! using the [`fetch::fetch_repo_data`] function: //! //! ```no_run -//! use std::{path::Path, default::Default}; +//! use std::{path::PathBuf, default::Default}; //! use rattler_networking::AuthenticatedClient; //! use url::Url; //! use rattler_repodata_gateway::fetch; @@ -35,7 +35,7 @@ //! async fn main() { //! let repodata_url = Url::parse("https://conda.anaconda.org/conda-forge/osx-64/").unwrap(); //! let client = AuthenticatedClient::default(); -//! let cache = Path::new("./cache"); +//! let cache = PathBuf::from("./cache"); //! //! let result = fetch::fetch_repo_data( //! repodata_url, diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 6a2ada256..21fb1360b 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -31,9 +31,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -80,6 +80,22 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-compression" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +dependencies = [ + "bzip2", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-executor" version = "1.5.1" @@ -161,7 +177,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -178,7 +194,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -210,9 +226,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -277,9 +293,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" @@ -287,18 +303,58 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "futures 0.1.31", + "libc", + "tokio-io", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cache_control" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf2a5fb3207c12b5d208ebc145f967fea5cac41a021c37417ccc31ba40f39ee" + [[package]] name = "cc" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -310,16 +366,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.29" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87d9d13be47a5b7c3907137f1290b0459a7f80efb26be8c52afb11963bccb02" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", - "time 0.1.45", "wasm-bindgen", "windows-targets", ] @@ -376,6 +431,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -416,7 +480,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -427,7 +491,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -506,14 +570,14 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] name = "enumflags2" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c041f5090df68b32bcd905365fd51769c8b9d553fe87fde0b683534f10c01bd2" +checksum = "5998b4f30320c9d93aed72f63af821bfdac50465b75428fce77b48ec482c3939" dependencies = [ "enumflags2_derive", "serde", @@ -521,13 +585,13 @@ dependencies = [ [[package]] name = "enumflags2_derive" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" +checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -578,12 +642,49 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "filetime" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.3.5", + "windows-sys", +] + +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -593,6 +694,27 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -600,6 +722,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -608,6 +731,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -637,7 +771,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -658,6 +792,7 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -697,7 +832,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -719,7 +854,7 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ - "bytes", + "bytes 1.5.0", "fnv", "futures-core", "futures-sink", @@ -752,15 +887,18 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "hkdf" @@ -786,7 +924,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.5.0", "fnv", "itoa", ] @@ -797,7 +935,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.5.0", "http", "pin-project-lite", ] @@ -814,13 +952,28 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes", + "bytes 1.5.0", "futures-channel", "futures-core", "futures-util", @@ -838,6 +991,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.5.0", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -931,6 +1097,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -952,6 +1127,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.64" @@ -961,6 +1145,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7765dccf8c39c3a470fc694efe322969d791e713ca46bc7b5c506886157572" +dependencies = [ + "serde", + "serde_json", + "thiserror", + "treediff", +] + [[package]] name = "keyring" version = "2.0.5" @@ -977,9 +1173,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57451d19ad5e289ff6c3d69c2a2424652995c42b79dafa11e9c4d5508c913c01" +checksum = "e723bd417b2df60a0f6a2b6825f297ea04b245d4ba52b5a22cb679bdf58b05fa" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -995,7 +1191,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1006,9 +1202,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libloading" @@ -1020,6 +1216,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + [[package]] name = "libz-sys" version = "1.1.12" @@ -1058,9 +1260,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -1080,10 +1282,11 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] @@ -1093,6 +1296,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -1139,10 +1351,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.26.4" @@ -1241,6 +1471,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.32.1" @@ -1262,6 +1502,50 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -1342,7 +1626,7 @@ dependencies = [ "line-wrap", "quick-xml", "serde", - "time 0.3.28", + "time", ] [[package]] @@ -1379,9 +1663,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -1390,13 +1674,19 @@ dependencies = [ name = "py-rattler" version = "0.1.0" dependencies = [ + "anyhow", + "futures 0.3.28", "pyo3", + "pyo3-asyncio", + "rattler", "rattler_conda_types", "rattler_networking", + "rattler_repodata_gateway", "rattler_shell", "rattler_solve", "rattler_virtual_packages", "thiserror", + "tokio", "url", ] @@ -1418,6 +1708,19 @@ dependencies = [ "unindent", ] +[[package]] +name = "pyo3-asyncio" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2cc34c1f907ca090d7add03dc523acdd91f3a4dab12286604951e2f5152edad" +dependencies = [ + "futures 0.3.28", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" version = "0.19.2" @@ -1509,6 +1812,45 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rattler" +version = "0.9.0" +dependencies = [ + "anyhow", + "async-compression", + "bytes 1.5.0", + "chrono", + "digest", + "dirs", + "futures 0.3.28", + "fxhash", + "hex", + "itertools", + "memchr", + "memmap2", + "nom", + "once_cell", + "pin-project-lite", + "rattler_conda_types", + "rattler_digest", + "rattler_networking", + "rattler_package_streaming", + "regex", + "reqwest", + "serde", + "serde_json", + "serde_with", + "smallvec", + "tempfile", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "uuid", +] + [[package]] name = "rattler_conda_types" version = "0.9.0" @@ -1548,11 +1890,12 @@ dependencies = [ "serde", "serde_with", "sha2", + "tokio", ] [[package]] name = "rattler_libsolv_c" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "cc", @@ -1567,7 +1910,7 @@ name = "rattler_macros" version = "0.9.0" dependencies = [ "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1588,6 +1931,60 @@ dependencies = [ "tracing", ] +[[package]] +name = "rattler_package_streaming" +version = "0.9.0" +dependencies = [ + "bzip2", + "chrono", + "futures-util", + "itertools", + "rattler_conda_types", + "rattler_digest", + "rattler_networking", + "reqwest", + "serde_json", + "tar", + "thiserror", + "tokio", + "tokio-util", + "url", + "zip", + "zstd", +] + +[[package]] +name = "rattler_repodata_gateway" +version = "0.9.0" +dependencies = [ + "anyhow", + "async-compression", + "blake2", + "cache_control", + "chrono", + "futures 0.3.28", + "hex", + "humansize", + "humantime", + "json-patch", + "libc", + "md-5", + "pin-project-lite", + "rattler_digest", + "rattler_networking", + "reqwest", + "serde", + "serde_json", + "serde_with", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "url", + "windows-sys", +] + [[package]] name = "rattler_shell" version = "0.9.0" @@ -1605,7 +2002,7 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "chrono", @@ -1624,7 +2021,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "0.8.0" +version = "0.9.0" dependencies = [ "cfg-if", "libloading", @@ -1702,8 +2099,9 @@ version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ + "async-compression", "base64", - "bytes", + "bytes 1.5.0", "encoding_rs", "futures-core", "futures-util", @@ -1711,10 +2109,12 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -1722,10 +2122,13 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] @@ -1763,14 +2166,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.5", + "linux-raw-sys 0.4.7", "windows-sys", ] @@ -1792,6 +2195,15 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1866,14 +2278,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "indexmap 2.0.0", "itoa", @@ -1889,7 +2301,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1918,7 +2330,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time 0.3.28", + "time", ] [[package]] @@ -1930,7 +2342,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1948,9 +2360,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -2004,9 +2416,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" dependencies = [ "serde", ] @@ -2023,9 +2435,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -2062,7 +2474,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -2084,15 +2496,26 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.11" @@ -2108,7 +2531,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.11", + "rustix 0.38.14", "windows-sys", ] @@ -2129,25 +2552,14 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] name = "time" -version = "0.1.45" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -2158,15 +2570,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -2193,21 +2605,66 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes", + "bytes 1.5.0", "libc", "mio", + "num_cpus", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.4", + "tokio-macros", "windows-sys", ] +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ - "bytes", + "bytes 1.5.0", "futures-core", "futures-sink", "pin-project-lite", @@ -2223,9 +2680,9 @@ checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.0.0", "toml_datetime", @@ -2258,7 +2715,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -2270,6 +2727,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "treediff" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" +dependencies = [ + "serde_json", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -2278,9 +2744,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uds_windows" @@ -2300,9 +2766,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2337,6 +2803,16 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +dependencies = [ + "getrandom", + "rand", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2364,12 +2840,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2397,7 +2867,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -2431,7 +2901,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2442,6 +2912,19 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -2568,6 +3051,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "xattr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +dependencies = [ + "libc", +] + [[package]] name = "xdg-home" version = "1.0.0" @@ -2644,6 +3136,49 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", + "time", +] + +[[package]] +name = "zstd" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "zvariant" version = "3.15.0" diff --git a/py-rattler/Cargo.toml b/py-rattler/Cargo.toml index ddcc319a5..1a201ca51 100644 --- a/py-rattler/Cargo.toml +++ b/py-rattler/Cargo.toml @@ -9,6 +9,11 @@ name = "rattler" crate-type = ["cdylib"] [dependencies] +anyhow = "1.0.75" +futures = "0.3.28" + +rattler = { path = "../crates/rattler" } +rattler_repodata_gateway = { path = "../crates/rattler_repodata_gateway" } rattler_conda_types = { path = "../crates/rattler_conda_types", default-features = false } rattler_networking = { path = "../crates/rattler_networking", default-features = false } rattler_shell = { path = "../crates/rattler_shell", default-features = false } @@ -20,6 +25,8 @@ pyo3 = { version = "0.19", features = [ "extension-module", "multiple-pymethods", ] } +pyo3-asyncio = { version = "0.19", features = ["tokio-runtime"] } +tokio = { version = "1.32" } thiserror = "1.0.44" url = "2.4.1" diff --git a/py-rattler/pixi.lock b/py-rattler/pixi.lock index cd3254ff0..7e25dd991 100644 --- a/py-rattler/pixi.lock +++ b/py-rattler/pixi.lock @@ -2331,6 +2331,98 @@ package: noarch: python size: 243695 timestamp: 1687692277221 +- name: pytest-asyncio + version: 0.21.1 + manager: conda + platform: linux-64 + dependencies: + pytest: '>=7.0.0' + python: '>=3.7' + typing_extensions: '>=3.7.2' + url: https://conda.anaconda.org/conda-forge/noarch/pytest-asyncio-0.21.1-pyhd8ed1ab_0.conda + hash: + md5: c8fd31da9b1059c4440f31f6ab4d620c + sha256: f061bd0f8a41886331d852f7b3c4b395ac40bc404283fd061dcf992f197e3bb1 + optional: false + category: main + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: linux-64 + build_number: 0 + license: Apache-2.0 + license_family: APACHE + noarch: python + size: 26068 + timestamp: 1689174411610 +- name: pytest-asyncio + version: 0.21.1 + manager: conda + platform: osx-64 + dependencies: + pytest: '>=7.0.0' + python: '>=3.7' + typing_extensions: '>=3.7.2' + url: https://conda.anaconda.org/conda-forge/noarch/pytest-asyncio-0.21.1-pyhd8ed1ab_0.conda + hash: + md5: c8fd31da9b1059c4440f31f6ab4d620c + sha256: f061bd0f8a41886331d852f7b3c4b395ac40bc404283fd061dcf992f197e3bb1 + optional: false + category: main + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: osx-64 + build_number: 0 + license: Apache-2.0 + license_family: APACHE + noarch: python + size: 26068 + timestamp: 1689174411610 +- name: pytest-asyncio + version: 0.21.1 + manager: conda + platform: osx-arm64 + dependencies: + pytest: '>=7.0.0' + python: '>=3.7' + typing_extensions: '>=3.7.2' + url: https://conda.anaconda.org/conda-forge/noarch/pytest-asyncio-0.21.1-pyhd8ed1ab_0.conda + hash: + md5: c8fd31da9b1059c4440f31f6ab4d620c + sha256: f061bd0f8a41886331d852f7b3c4b395ac40bc404283fd061dcf992f197e3bb1 + optional: false + category: main + build: pyhd8ed1ab_0 + arch: aarch64 + subdir: osx-arm64 + build_number: 0 + license: Apache-2.0 + license_family: APACHE + noarch: python + size: 26068 + timestamp: 1689174411610 +- name: pytest-asyncio + version: 0.21.1 + manager: conda + platform: win-64 + dependencies: + pytest: '>=7.0.0' + python: '>=3.7' + typing_extensions: '>=3.7.2' + url: https://conda.anaconda.org/conda-forge/noarch/pytest-asyncio-0.21.1-pyhd8ed1ab_0.conda + hash: + md5: c8fd31da9b1059c4440f31f6ab4d620c + sha256: f061bd0f8a41886331d852f7b3c4b395ac40bc404283fd061dcf992f197e3bb1 + optional: false + category: main + build: pyhd8ed1ab_0 + arch: x86_64 + subdir: win-64 + build_number: 0 + license: Apache-2.0 + license_family: APACHE + noarch: python + size: 26068 + timestamp: 1689174411610 - name: python version: 3.11.4 manager: conda diff --git a/py-rattler/pixi.toml b/py-rattler/pixi.toml index 16f04b231..dafb3165b 100644 --- a/py-rattler/pixi.toml +++ b/py-rattler/pixi.toml @@ -33,3 +33,4 @@ pytest = "~=7.4.0" black = "~=23.7.0" ruff = "~=0.0.285" mypy = "~=1.5.1" +pytest-asyncio = "0.21.1.*" diff --git a/py-rattler/rattler/__init__.py b/py-rattler/rattler/__init__.py index 1a079767e..2ed5f993c 100644 --- a/py-rattler/rattler/__init__.py +++ b/py-rattler/rattler/__init__.py @@ -1,8 +1,8 @@ from rattler.version import Version, VersionWithSource from rattler.match_spec import MatchSpec, NamelessMatchSpec -from rattler.repo_data import PackageRecord +from rattler.repo_data import PackageRecord, RepoData, RepoDataRecord, PatchInstructions from rattler.channel import Channel, ChannelConfig -from rattler.networking import AuthenticatedClient +from rattler.networking import AuthenticatedClient, fetch_repo_data from rattler.virtual_package import GenericVirtualPackage, VirtualPackage from rattler.package import PackageName from rattler.prefix import PrefixRecord, PrefixPaths @@ -16,6 +16,10 @@ "Channel", "ChannelConfig", "AuthenticatedClient", + "PatchInstructions", + "RepoDataRecord", + "RepoData", + "fetch_repo_data", "GenericVirtualPackage", "VirtualPackage", "PackageName", diff --git a/py-rattler/rattler/match_spec/match_spec.py b/py-rattler/rattler/match_spec/match_spec.py index ecd80c5b7..2f792c0dc 100644 --- a/py-rattler/rattler/match_spec/match_spec.py +++ b/py-rattler/rattler/match_spec/match_spec.py @@ -121,7 +121,13 @@ def from_nameless(cls, spec: NamelessMatchSpec, name: str) -> Self: ) def __str__(self) -> str: + """ + Returns a string representation of the MatchSpec. + """ return self._match_spec.as_str() def __repr__(self) -> str: + """ + Returns a representation of the MatchSpec. + """ return f'MatchSpec("{self._match_spec.as_str()}")' diff --git a/py-rattler/rattler/match_spec/nameless_match_spec.py b/py-rattler/rattler/match_spec/nameless_match_spec.py index e69f29b35..c295d0ff1 100644 --- a/py-rattler/rattler/match_spec/nameless_match_spec.py +++ b/py-rattler/rattler/match_spec/nameless_match_spec.py @@ -52,7 +52,13 @@ def from_match_spec(cls, spec: MatchSpec) -> Self: ) def __str__(self) -> str: + """ + Returns a string representation of the NamelessMatchSpec. + """ return self._nameless_match_spec.as_str() def __repr__(self) -> str: + """ + Returns a representation of the NamelessMatchSpec. + """ return f'NamelessMatchSpec("{self._nameless_match_spec.as_str()}")' diff --git a/py-rattler/rattler/networking/__init__.py b/py-rattler/rattler/networking/__init__.py index 36af1c6ff..3676bf7bb 100644 --- a/py-rattler/rattler/networking/__init__.py +++ b/py-rattler/rattler/networking/__init__.py @@ -1,3 +1,4 @@ from rattler.networking.authenticated_client import AuthenticatedClient +from rattler.networking.fetch_repo_data import fetch_repo_data -__all__ = ["AuthenticatedClient"] +__all__ = ["AuthenticatedClient", "fetch_repo_data"] diff --git a/py-rattler/rattler/networking/fetch_repo_data.py b/py-rattler/rattler/networking/fetch_repo_data.py new file mode 100644 index 000000000..be212c395 --- /dev/null +++ b/py-rattler/rattler/networking/fetch_repo_data.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import Callable, List, Optional, Union, TYPE_CHECKING + + +from rattler.rattler import py_fetch_repo_data +from rattler.repo_data import RepoData + +if TYPE_CHECKING: + import os + from rattler.channel import Channel + from rattler.platform import Platform + + +async def fetch_repo_data( + *, + channels: List[Channel], + platforms: List[Platform], + cache_path: Union[str, os.PathLike[str]], + callback: Optional[Callable[[int, int], None]], +) -> List[RepoData]: + """ + Returns a list of RepoData for given channels and platform. + """ + repo_data_list = await py_fetch_repo_data( + [channel._channel for channel in channels], + [platform._inner for platform in platforms], + cache_path, + callback, + ) + + return [RepoData._from_py_repo_data(repo_data) for repo_data in repo_data_list] diff --git a/py-rattler/rattler/prefix/prefix_paths.py b/py-rattler/rattler/prefix/prefix_paths.py index 21cad3167..7d92282ff 100644 --- a/py-rattler/rattler/prefix/prefix_paths.py +++ b/py-rattler/rattler/prefix/prefix_paths.py @@ -51,7 +51,7 @@ def paths(self) -> List[os.PathLike[str]]: def __repr__(self) -> str: """ - Returns a representation of the version. + Returns a representation of the PrefixPaths. Examples -------- diff --git a/py-rattler/rattler/repo_data/__init__.py b/py-rattler/rattler/repo_data/__init__.py index c2e09d680..78ff83510 100644 --- a/py-rattler/rattler/repo_data/__init__.py +++ b/py-rattler/rattler/repo_data/__init__.py @@ -1,3 +1,6 @@ from rattler.repo_data.package_record import PackageRecord +from rattler.repo_data.repo_data import RepoData +from rattler.repo_data.patch_instructions import PatchInstructions +from rattler.repo_data.record import RepoDataRecord -__all__ = ["PackageRecord"] +__all__ = ["PackageRecord", "RepoData", "PatchInstructions", "RepoDataRecord"] diff --git a/py-rattler/rattler/repo_data/package_record.py b/py-rattler/rattler/repo_data/package_record.py index 10d014978..5072de2cb 100644 --- a/py-rattler/rattler/repo_data/package_record.py +++ b/py-rattler/rattler/repo_data/package_record.py @@ -1,4 +1,5 @@ from __future__ import annotations +from typing import Self from rattler.rattler import PyPackageRecord @@ -13,8 +14,23 @@ class PackageRecord: def __init__(self) -> None: self._package_record = PyPackageRecord() + @classmethod + def _from_py_package_record(cls, py_package_record: PyPackageRecord) -> Self: + """ + Construct Rattler PackageRecord from FFI PyPackageRecord object. + """ + package_record = cls.__new__(cls) + package_record._package_record = py_package_record + return package_record + def __str__(self) -> str: + """ + Returns the string representation of the PackageRecord. + """ return self._package_record.as_str() def __repr__(self) -> str: - return self.__str__() + """ + Returns a representation of the PackageRecord. + """ + return f'PackageRecord("{self.__str__()}")' diff --git a/py-rattler/rattler/repo_data/patch_instructions.py b/py-rattler/rattler/repo_data/patch_instructions.py new file mode 100644 index 000000000..be780a378 --- /dev/null +++ b/py-rattler/rattler/repo_data/patch_instructions.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from typing import Self + +from rattler.rattler import PyPatchInstructions + + +class PatchInstructions: + _patch_instructions: PyPatchInstructions + + @classmethod + def _from_py_patch_instructions( + cls, py_patch_instructions: PyPatchInstructions + ) -> Self: + """ + Construct Rattler PatchInstructions from FFI PyPatchInstructions object. + """ + patch_instructions = cls.__new__(cls) + patch_instructions._patch_instructions = py_patch_instructions + return patch_instructions + + def __repr__(self) -> str: + """ + Returns a representation of the PatchInstructions. + """ + return f"{type(self).__name__}()" diff --git a/py-rattler/rattler/repo_data/record.py b/py-rattler/rattler/repo_data/record.py new file mode 100644 index 000000000..af1c05a47 --- /dev/null +++ b/py-rattler/rattler/repo_data/record.py @@ -0,0 +1,54 @@ +from __future__ import annotations +from typing import Self + +from rattler.rattler import PyRepoDataRecord +from rattler.repo_data import PackageRecord + + +class RepoDataRecord: + _record: PyRepoDataRecord + + @property + def package_record(self) -> PackageRecord: + """ + The data stored in the repodata.json. + """ + return PackageRecord._from_py_package_record(self._record.package_record) + + @property + def url(self) -> str: + """ + The canonical URL from where to get this package. + """ + return self._record.url + + @property + def channel(self) -> str: + """ + String representation of the channel where the + package comes from. This could be a URL but it + could also be a channel name. + """ + return self._record.channel + + @property + def file_name(self) -> str: + """ + The filename of the package. + """ + return self._record.file_name + + @classmethod + def _from_py_record(cls, py_record: PyRepoDataRecord) -> Self: + """ + Construct Rattler RepoDataRecord from FFI PyRepoDataRecord object. + """ + record = cls.__new__(cls) + record._record = py_record + return record + + def __repr__(self) -> str: + """ + Returns a representation of the RepoDataRecord. + """ + return f"{type(self).__name__}()" diff --git a/py-rattler/rattler/repo_data/repo_data.py b/py-rattler/rattler/repo_data/repo_data.py new file mode 100644 index 000000000..584876242 --- /dev/null +++ b/py-rattler/rattler/repo_data/repo_data.py @@ -0,0 +1,58 @@ +from __future__ import annotations +from pathlib import Path +from typing import Self, Union, List, TYPE_CHECKING + + +if TYPE_CHECKING: + from os import PathLike + from rattler.channel import Channel + from rattler.repo_data import PatchInstructions, RepoDataRecord + +from rattler.rattler import PyRepoData + + +class RepoData: + def __init__(self, path: Union[str, PathLike[str]]) -> None: + if not isinstance(path, (str, Path)): + raise TypeError( + "RepoData constructor received unsupported type " + f" {type(path).__name__!r} for the `path` parameter" + ) + + self._repo_data = PyRepoData.from_path(path) + + def apply_patches(self, instructions: PatchInstructions) -> None: + """ + Apply a patch to a repodata file. + Note that we currently do not handle revoked instructions. + """ + self._repo_data.apply_patches(instructions._patch_instructions) + + def into_repo_data(self, channel: Channel) -> List[RepoDataRecord]: + """ + Builds a `List[RepoDataRecord]` from the packages in a + `RepoData` given the source of the data. + """ + from rattler.repo_data import RepoDataRecord + + return [ + RepoDataRecord._from_py_record(record) + for record in PyRepoData.repo_data_to_records( + self._repo_data, channel._channel + ) + ] + + @classmethod + def _from_py_repo_data(cls, py_repo_data: PyRepoData) -> Self: + """ + Construct Rattler RepoData from FFI PyRepoData object. + """ + repo_data = cls.__new__(cls) + repo_data._repo_data = py_repo_data + return repo_data + + def __repr__(self) -> str: + """ + Returns a representation of the RepoData. + """ + return f"{type(self).__name__}()" diff --git a/py-rattler/src/channel/mod.rs b/py-rattler/src/channel/mod.rs index fa4ac648c..317de685d 100644 --- a/py-rattler/src/channel/mod.rs +++ b/py-rattler/src/channel/mod.rs @@ -2,7 +2,7 @@ use pyo3::{pyclass, pymethods}; use rattler_conda_types::{Channel, ChannelConfig}; use url::Url; -use crate::error::PyRattlerError; +use crate::{error::PyRattlerError, platform::PyPlatform}; #[pyclass] #[repr(transparent)] @@ -22,7 +22,7 @@ impl PyChannelConfig { }) } - /// Return the channel alias that is configured + /// Returns the channel alias that is configured #[getter] fn channel_alias(&self) -> String { self.inner.channel_alias.to_string() @@ -57,15 +57,20 @@ impl PyChannel { .map_err(PyRattlerError::from)?) } - /// Return the name of the channel + /// Returns the name of the channel. #[getter] fn name(&self) -> Option { self.inner.name.clone() } - /// Return the base url of the channel + /// Returns the base url of the channel. #[getter] fn base_url(&self) -> String { self.inner.base_url.to_string() } + + /// Returns the Urls for the given platform. + pub fn platform_url(&self, platform: &PyPlatform) -> String { + self.inner.platform_url(platform.clone().into()).into() + } } diff --git a/py-rattler/src/error.rs b/py-rattler/src/error.rs index 5cdf7cdd9..b806e7e22 100644 --- a/py-rattler/src/error.rs +++ b/py-rattler/src/error.rs @@ -6,6 +6,7 @@ use rattler_conda_types::{ InvalidPackageNameError, ParseArchError, ParseChannelError, ParseMatchSpecError, ParsePlatformError, ParseVersionError, }; +use rattler_repodata_gateway::fetch::FetchRepoDataError; use rattler_shell::activation::ActivationError; use rattler_virtual_packages::DetectVirtualPackageError; use thiserror::Error; @@ -30,6 +31,10 @@ pub enum PyRattlerError { #[error(transparent)] ParseArchError(#[from] ParseArchError), #[error(transparent)] + FetchRepoDataError(#[from] FetchRepoDataError), + #[error(transparent)] + CacheDirError(#[from] anyhow::Error), + #[error(transparent)] DetectVirtualPackageError(#[from] DetectVirtualPackageError), #[error(transparent)] IoError(#[from] io::Error), @@ -56,7 +61,10 @@ impl From for PyErr { ParsePlatformException::new_err(err.to_string()) } PyRattlerError::ParseArchError(err) => ParseArchException::new_err(err.to_string()), - + PyRattlerError::FetchRepoDataError(err) => { + FetchRepoDataException::new_err(err.to_string()) + } + PyRattlerError::CacheDirError(err) => CacheDirException::new_err(err.to_string()), PyRattlerError::DetectVirtualPackageError(err) => { DetectVirtualPackageException::new_err(err.to_string()) } @@ -73,5 +81,7 @@ create_exception!(exceptions, InvalidChannelException, PyException); create_exception!(exceptions, ActivationException, PyException); create_exception!(exceptions, ParsePlatformException, PyException); create_exception!(exceptions, ParseArchException, PyException); +create_exception!(exceptions, FetchRepoDataException, PyException); +create_exception!(exceptions, CacheDirException, PyException); create_exception!(exceptions, DetectVirtualPackageException, PyException); create_exception!(exceptions, IoException, PyException); diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index 7d744bd71..f3d943cab 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -21,10 +21,13 @@ use error::{ use generic_virtual_package::PyGenericVirtualPackage; use match_spec::PyMatchSpec; use nameless_match_spec::PyNamelessMatchSpec; -use networking::PyAuthenticatedClient; +use networking::{authenticated_client::PyAuthenticatedClient, py_fetch_repo_data}; use package_name::PyPackageName; use prefix_record::{PyPrefixPaths, PyPrefixRecord}; -use repo_data::package_record::PyPackageRecord; +use repo_data::{ + package_record::PyPackageRecord, patch_instructions::PyPatchInstructions, + repo_data_record::PyRepoDataRecord, PyRepoData, +}; use version::PyVersion; use pyo3::prelude::*; @@ -56,6 +59,12 @@ fn rattler(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::().unwrap(); m.add_class::().unwrap(); + m.add_class::().unwrap(); + m.add_class::().unwrap(); + m.add_class::().unwrap(); + + m.add_function(wrap_pyfunction!(py_fetch_repo_data, m).unwrap()) + .unwrap(); m.add_class::().unwrap(); m.add_class::().unwrap(); m.add_class::().unwrap(); diff --git a/py-rattler/src/networking/authenticated_client.rs b/py-rattler/src/networking/authenticated_client.rs new file mode 100644 index 000000000..521f0563f --- /dev/null +++ b/py-rattler/src/networking/authenticated_client.rs @@ -0,0 +1,35 @@ +use pyo3::{pyclass, pymethods}; +use rattler_networking::AuthenticatedClient; + +#[pyclass] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyAuthenticatedClient { + pub(crate) inner: AuthenticatedClient, +} + +#[pymethods] +impl PyAuthenticatedClient { + #[new] + pub fn new() -> Self { + Self::default() + } +} + +impl From for PyAuthenticatedClient { + fn from(value: AuthenticatedClient) -> Self { + Self { inner: value } + } +} + +impl From for AuthenticatedClient { + fn from(value: PyAuthenticatedClient) -> Self { + value.inner + } +} + +impl Default for PyAuthenticatedClient { + fn default() -> Self { + AuthenticatedClient::default().into() + } +} diff --git a/py-rattler/src/networking/cached_repo_data.rs b/py-rattler/src/networking/cached_repo_data.rs new file mode 100644 index 000000000..f8b4e9ceb --- /dev/null +++ b/py-rattler/src/networking/cached_repo_data.rs @@ -0,0 +1,33 @@ +use pyo3::{pyclass, pymethods}; +use rattler_repodata_gateway::fetch::CachedRepoData; +use std::sync::Arc; + +#[pyclass] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyCachedRepoData { + pub(crate) inner: Arc, +} + +impl From for CachedRepoData { + fn from(value: PyCachedRepoData) -> Self { + Arc::::into_inner(value.inner) + .expect("CachedRepoData has multiple strong references!") + } +} + +impl From for PyCachedRepoData { + fn from(value: CachedRepoData) -> Self { + Self { + inner: Arc::new(value), + } + } +} + +#[pymethods] +impl PyCachedRepoData { + /// Returns a string representation of PyCachedRepoData. + pub fn as_str(&self) -> String { + format!("{:?}", self.inner) + } +} diff --git a/py-rattler/src/networking/mod.rs b/py-rattler/src/networking/mod.rs index 521f0563f..a8da9c047 100644 --- a/py-rattler/src/networking/mod.rs +++ b/py-rattler/src/networking/mod.rs @@ -1,35 +1,84 @@ -use pyo3::{pyclass, pymethods}; -use rattler_networking::AuthenticatedClient; - -#[pyclass] -#[repr(transparent)] -#[derive(Clone)] -pub struct PyAuthenticatedClient { - pub(crate) inner: AuthenticatedClient, -} +use futures::future::try_join_all; +use pyo3::{pyfunction, types::PyTuple, Py, PyAny, PyResult, Python, ToPyObject}; +use pyo3_asyncio::tokio::future_into_py; -#[pymethods] -impl PyAuthenticatedClient { - #[new] - pub fn new() -> Self { - Self::default() - } -} +use rattler_repodata_gateway::fetch::{fetch_repo_data, DownloadProgress, FetchRepoDataOptions}; +use url::Url; + +use std::{path::PathBuf, str::FromStr}; + +use crate::{ + channel::PyChannel, error::PyRattlerError, platform::PyPlatform, repo_data::PyRepoData, +}; +use authenticated_client::PyAuthenticatedClient; + +pub mod authenticated_client; +pub mod cached_repo_data; -impl From for PyAuthenticatedClient { - fn from(value: AuthenticatedClient) -> Self { - Self { inner: value } +/// High-level function to fetch repodata for all the subdirectory of channels and platform. +/// Returns a list of `PyRepoData`. +#[pyfunction] +pub fn py_fetch_repo_data<'a>( + py: Python<'a>, + channels: Vec, + platforms: Vec, + cache_path: PathBuf, + callback: Option<&'a PyAny>, +) -> PyResult<&'a PyAny> { + let mut meta_futures = Vec::new(); + let client = PyAuthenticatedClient::new(); + + for subdir in get_subdir_urls(channels, platforms)? { + let progress = if let Some(callback) = callback { + let callback = callback.to_object(py); + Some(get_progress_func(callback)) + } else { + None + }; + let client = client.clone(); + + // Push all the future into meta_future vec to be resolve later + meta_futures.push(fetch_repo_data( + subdir, + client.into(), + cache_path.clone(), + FetchRepoDataOptions::default(), + progress, + )); } + + future_into_py(py, async move { + // Resolve all the meta_futures together + match try_join_all(meta_futures).await { + Ok(cached_vec) => cached_vec + .into_iter() + .map(|c| PyRepoData::from_path(c.repo_data_json_path)) + .collect::, _>>(), + Err(e) => Err(PyRattlerError::from(e).into()), + } + }) } -impl From for AuthenticatedClient { - fn from(value: PyAuthenticatedClient) -> Self { - value.inner - } +/// Creates a closure to show progress of Download +fn get_progress_func(callback: Py) -> Box { + Box::new(move |progress: DownloadProgress| { + Python::with_gil(|py| { + let args = PyTuple::new(py, [Some(progress.bytes), progress.total]); + callback.call1(py, args).expect("Callback failed!"); + }); + }) } -impl Default for PyAuthenticatedClient { - fn default() -> Self { - AuthenticatedClient::default().into() +/// Creates a subdir urls out of channels and channels. +fn get_subdir_urls(channels: Vec, platforms: Vec) -> PyResult> { + let mut urls = Vec::new(); + + for c in channels { + for p in &platforms { + let r = c.platform_url(p); + urls.push(Url::from_str(r.as_str()).map_err(PyRattlerError::from)?); + } } + + Ok(urls) } diff --git a/py-rattler/src/repo_data/mod.rs b/py-rattler/src/repo_data/mod.rs index da0d5b5f7..8e3760c47 100644 --- a/py-rattler/src/repo_data/mod.rs +++ b/py-rattler/src/repo_data/mod.rs @@ -1 +1,67 @@ +use std::path::PathBuf; + +use pyo3::{pyclass, pymethods, PyResult}; +use rattler_conda_types::RepoData; + +use crate::{channel::PyChannel, error::PyRattlerError}; + +use patch_instructions::PyPatchInstructions; +use repo_data_record::PyRepoDataRecord; + pub mod package_record; +pub mod patch_instructions; +pub mod repo_data_record; + +#[pyclass] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyRepoData { + pub(crate) inner: RepoData, +} + +impl From for RepoData { + fn from(value: PyRepoData) -> Self { + value.inner + } +} + +impl From for PyRepoData { + fn from(value: RepoData) -> Self { + Self { inner: value } + } +} + +#[pymethods] +impl PyRepoData { + /// Apply a patch to a repodata file Note that we currently do not handle revoked instructions. + pub fn apply_patches(&mut self, instructions: &PyPatchInstructions) { + self.inner.apply_patches(&instructions.inner) + } + + /// Gets the string representation of the `PyRepoData`. + pub fn as_str(&self) -> String { + format!("{:?}", self.inner) + } +} + +#[pymethods] +impl PyRepoData { + /// Parses RepoData from a file. + #[staticmethod] + pub fn from_path(path: PathBuf) -> PyResult { + Ok(RepoData::from_path(path) + .map(Into::into) + .map_err(PyRattlerError::IoError)?) + } + + /// Builds a `Vec` from the packages in a `PyRepoData` given the source of the data. + #[staticmethod] + pub fn repo_data_to_records(repo_data: Self, channel: &PyChannel) -> Vec { + repo_data + .inner + .into_repo_data_records(&channel.inner) + .into_iter() + .map(Into::into) + .collect() + } +} diff --git a/py-rattler/src/repo_data/patch_instructions.rs b/py-rattler/src/repo_data/patch_instructions.rs new file mode 100644 index 000000000..3c3ee05fb --- /dev/null +++ b/py-rattler/src/repo_data/patch_instructions.rs @@ -0,0 +1,9 @@ +use pyo3::pyclass; +use rattler_conda_types::PatchInstructions; + +#[pyclass] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyPatchInstructions { + pub(crate) inner: PatchInstructions, +} diff --git a/py-rattler/src/repo_data/repo_data_record.rs b/py-rattler/src/repo_data/repo_data_record.rs new file mode 100644 index 000000000..2ad6e0bbd --- /dev/null +++ b/py-rattler/src/repo_data/repo_data_record.rs @@ -0,0 +1,57 @@ +use pyo3::{pyclass, pymethods}; +use rattler_conda_types::RepoDataRecord; + +use super::package_record::PyPackageRecord; + +#[pyclass] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyRepoDataRecord { + pub(crate) inner: RepoDataRecord, +} + +impl From for RepoDataRecord { + fn from(value: PyRepoDataRecord) -> Self { + value.inner + } +} + +impl From for PyRepoDataRecord { + fn from(value: RepoDataRecord) -> Self { + Self { inner: value } + } +} + +#[pymethods] +impl PyRepoDataRecord { + /// The data stored in the repodata.json. + #[getter] + pub fn package_record(&self) -> PyPackageRecord { + self.inner.package_record.clone().into() + } + + /// The filename of the package. + #[getter] + pub fn file_name(&self) -> String { + self.inner.file_name.clone() + } + + /// The canonical URL from where to get this package. + #[getter] + pub fn url(&self) -> String { + self.inner.url.to_string() + } + + /// String representation of the channel where the + /// package comes from. This could be a URL but it + /// could also be a channel name. + #[getter] + pub fn channel(&self) -> String { + self.inner.channel.clone() + } + + /// Returns a string representation of PyRepoDataRecord. + pub fn as_str(&self) -> String { + format!("{:?}", self.inner) + } +} diff --git a/py-rattler/tests/unit/test_fetch_repo_data.py b/py-rattler/tests/unit/test_fetch_repo_data.py new file mode 100644 index 000000000..f72458bc8 --- /dev/null +++ b/py-rattler/tests/unit/test_fetch_repo_data.py @@ -0,0 +1,65 @@ +# type: ignore +import os.path +import subprocess + +import pytest +from rattler import Channel, ChannelConfig, fetch_repo_data, RepoData +from rattler.platform import Platform +from rattler.repo_data.record import RepoDataRecord + + +@pytest.fixture(scope="session") +def serve_repo_data() -> None: + port, repo_name = 8912, "test-repo" + + test_data_dir = os.path.join( + os.path.dirname(__file__), "../../../test-data/test-server" + ) + + with subprocess.Popen( + [ + "python", + os.path.join(test_data_dir, "reposerver.py"), + "-d", + os.path.join(test_data_dir, "repo"), + "-n", + repo_name, + "-p", + str(port), + ] + ) as proc: + yield port, repo_name + proc.terminate() + + +@pytest.mark.asyncio +async def test_fetch_repo_data( + tmp_path, + serve_repo_data, +): + port, repo = serve_repo_data + cache_dir = tmp_path / "test_repo_data_download" + chan = Channel(repo, ChannelConfig(f"http://localhost:{port}/")) + plat = Platform("noarch") + + result = await fetch_repo_data( + channels=[chan], + platforms=[plat], + cache_path=cache_dir, + callback=None, + ) + assert isinstance(result, list) + + repodata = result[0] + assert isinstance(repodata, RepoData) + + repodata_record = repodata.into_repo_data(chan)[0] + assert isinstance(repodata_record, RepoDataRecord) + + assert repodata_record.channel == f"http://localhost:{port}/{repo}/" + assert repodata_record.file_name == "test-package-0.1-0.tar.bz2" + assert str(repodata_record.package_record) == "test-package=0.1=0" + assert ( + repodata_record.url + == f"http://localhost:{port}/test-repo/noarch/test-package-0.1-0.tar.bz2" + ) diff --git a/test-data/test-server/README.md b/test-data/test-server/README.md new file mode 100644 index 000000000..b9a9f9699 --- /dev/null +++ b/test-data/test-server/README.md @@ -0,0 +1,5 @@ +# test-server + +A simple server to serve repodata for test purposes. +Originally implemented by [mamba](https://github.com/mamba-org/mamba/tree/a8d595b6ff8ac182e60741c3e8cbd142e7d19905/mamba/tests) +under the BSD-3-Clause license. diff --git a/test-data/test-server/repo/channeldata.json b/test-data/test-server/repo/channeldata.json new file mode 100644 index 000000000..f73272e5f --- /dev/null +++ b/test-data/test-server/repo/channeldata.json @@ -0,0 +1,38 @@ +{ + "channeldata_version": 1, + "packages": { + "test-package": { + "activate.d": false, + "binary_prefix": false, + "deactivate.d": false, + "description": null, + "dev_url": null, + "doc_source_url": null, + "doc_url": null, + "home": "https://github.com/mamba-org/mamba", + "icon_hash": null, + "icon_url": null, + "identifiers": null, + "keywords": null, + "license": "BSD", + "post_link": false, + "pre_link": false, + "pre_unlink": false, + "recipe_origin": null, + "run_exports": {}, + "source_git_url": null, + "source_url": null, + "subdirs": [ + "noarch" + ], + "summary": "I am just a test package!", + "tags": null, + "text_prefix": false, + "timestamp": 1613117294, + "version": "0.1" + } + }, + "subdirs": [ + "noarch" + ] +} diff --git a/test-data/test-server/repo/index.html b/test-data/test-server/repo/index.html new file mode 100644 index 000000000..e4ded9e2d --- /dev/null +++ b/test-data/test-server/repo/index.html @@ -0,0 +1,90 @@ + + + repo + + + +

repo

+

RSS Feed   channeldata.json

+noarch    + + + + + + + + + + + + + + + +
PackageLatest VersionDocDevLicensenoarch Summary
test-package0.1BSDX I am just a test package!
+
Updated: 2021-02-12 09:02:37 +0000 - Files: 1
+ + diff --git a/test-data/test-server/repo/noarch/current_repodata.json b/test-data/test-server/repo/noarch/current_repodata.json new file mode 100644 index 000000000..52facdf7e --- /dev/null +++ b/test-data/test-server/repo/noarch/current_repodata.json @@ -0,0 +1,25 @@ +{ + "info": { + "subdir": "noarch" + }, + "packages": { + "test-package-0.1-0.tar.bz2": { + "build": "0", + "build_number": 0, + "depends": [], + "license": "BSD", + "license_family": "BSD", + "md5": "2a8595f37faa2950e1b433acbe91d481", + "name": "test-package", + "noarch": "generic", + "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988", + "size": 5719, + "subdir": "noarch", + "timestamp": 1613117294885, + "version": "0.1" + } + }, + "packages.conda": {}, + "removed": [], + "repodata_version": 1 +} diff --git a/test-data/test-server/repo/noarch/current_repodata.json.bz2 b/test-data/test-server/repo/noarch/current_repodata.json.bz2 new file mode 100644 index 000000000..76c130c02 Binary files /dev/null and b/test-data/test-server/repo/noarch/current_repodata.json.bz2 differ diff --git a/test-data/test-server/repo/noarch/index.html b/test-data/test-server/repo/noarch/index.html new file mode 100644 index 000000000..6c83d9446 --- /dev/null +++ b/test-data/test-server/repo/noarch/index.html @@ -0,0 +1,88 @@ + + + repo/noarch + + + +

repo/noarch

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FilenameSizeLast ModifiedSHA256MD5
repodata.json586 B2021-02-12 09:01:48 +0000cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b047501ec77771889b42a39c615158cb9c4
repodata.json.bz2351 B2021-02-12 09:01:48 +00009a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb450c926155642f0e894d97dc8a5af7007b
repodata_from_packages.json586 B2021-02-12 09:01:48 +0000cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b047501ec77771889b42a39c615158cb9c4
repodata_from_packages.json.bz2351 B2021-02-12 09:01:48 +00009a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb450c926155642f0e894d97dc8a5af7007b
test-package-0.1-0.tar.bz26 KB2021-02-12 08:08:14 +0000b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f69882a8595f37faa2950e1b433acbe91d481
+
Updated: 2021-02-12 09:02:37 +0000 - Files: 1
+ + diff --git a/test-data/test-server/repo/noarch/repodata.json b/test-data/test-server/repo/noarch/repodata.json new file mode 100644 index 000000000..83ce3c7f5 --- /dev/null +++ b/test-data/test-server/repo/noarch/repodata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b04 +size 586 diff --git a/test-data/test-server/repo/noarch/repodata.json.bz2 b/test-data/test-server/repo/noarch/repodata.json.bz2 new file mode 100644 index 000000000..76c130c02 Binary files /dev/null and b/test-data/test-server/repo/noarch/repodata.json.bz2 differ diff --git a/test-data/test-server/repo/noarch/repodata_from_packages.json b/test-data/test-server/repo/noarch/repodata_from_packages.json new file mode 100644 index 000000000..52facdf7e --- /dev/null +++ b/test-data/test-server/repo/noarch/repodata_from_packages.json @@ -0,0 +1,25 @@ +{ + "info": { + "subdir": "noarch" + }, + "packages": { + "test-package-0.1-0.tar.bz2": { + "build": "0", + "build_number": 0, + "depends": [], + "license": "BSD", + "license_family": "BSD", + "md5": "2a8595f37faa2950e1b433acbe91d481", + "name": "test-package", + "noarch": "generic", + "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988", + "size": 5719, + "subdir": "noarch", + "timestamp": 1613117294885, + "version": "0.1" + } + }, + "packages.conda": {}, + "removed": [], + "repodata_version": 1 +} diff --git a/test-data/test-server/repo/noarch/repodata_from_packages.json.bz2 b/test-data/test-server/repo/noarch/repodata_from_packages.json.bz2 new file mode 100644 index 000000000..76c130c02 Binary files /dev/null and b/test-data/test-server/repo/noarch/repodata_from_packages.json.bz2 differ diff --git a/test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2 b/test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2 new file mode 100644 index 000000000..d2b1fd405 --- /dev/null +++ b/test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988 +size 5719 diff --git a/test-data/test-server/repo/recipes/test-package/meta.yaml b/test-data/test-server/repo/recipes/test-package/meta.yaml new file mode 100644 index 000000000..8a141cf04 --- /dev/null +++ b/test-data/test-server/repo/recipes/test-package/meta.yaml @@ -0,0 +1,14 @@ +package: + name: test-package + version: 0.1 + +build: + number: 0 + script: echo Hello world + noarch: generic + +about: + home: https://github.com/mamba-org/mamba + license: BSD + license_family: BSD + summary: I am just a test package! diff --git a/test-data/test-server/reposerver.py b/test-data/test-server/reposerver.py new file mode 100644 index 000000000..56bd2deb2 --- /dev/null +++ b/test-data/test-server/reposerver.py @@ -0,0 +1,395 @@ +# File taken from https://github.com/mamba-org/mamba/tree/a8d595b6ff8ac182e60741c3e8cbd142e7d19905/mamba/tests +# under BSD-3-Clause license + +import argparse +import base64 +import glob +import os +import re +import shutil +import sys +from http.server import HTTPServer, SimpleHTTPRequestHandler +from pathlib import Path +from typing import Dict, List + +try: + import conda_content_trust.authentication as cct_authentication + import conda_content_trust.common as cct_common + import conda_content_trust.metadata_construction as cct_metadata_construction + import conda_content_trust.root_signing as cct_root_signing + import conda_content_trust.signing as cct_signing + + conda_content_trust_available = True +except ImportError: + conda_content_trust_available = False + + +def fatal_error(message: str) -> None: + """Print error and exit.""" + print(message, file=sys.stderr) + exit(1) + + +def get_fingerprint(gpg_output: str) -> str: + lines = gpg_output.splitlines() + fpline = lines[1].strip() + fpline = fpline.replace(" ", "") + return fpline + + +KeySet = Dict[str, List[Dict[str, str]]] + + +def normalize_keys(keys: KeySet) -> KeySet: + out = {} + for ik, iv in keys.items(): + out[ik] = [] + for el in iv: + if isinstance(el, str): + el = el.lower() + keyval = cct_root_signing.fetch_keyval_from_gpg(el) + res = {"fingerprint": el, "public": keyval} + elif isinstance(el, dict): + res = { + "private": el["private"].lower(), + "public": el["public"].lower(), + } + out[ik].append(res) + + return out + + +class RepoSigner: + keys = { + "root": [], + "key_mgr": [ + { + "private": "c9c2060d7e0d93616c2654840b4983d00221d8b6b69c850107da74b42168f937", + "public": "013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7", + }, + ], + "pkg_mgr": [ + { + "private": "f3cdab14740066fb277651ec4f96b9f6c3e3eb3f812269797b9656074cd52133", + "public": "f46b5a7caa43640744186564c098955147daa8bac4443887bc64d8bfee3d3569", + } + ], + } + + def __init__(self, in_folder: str) -> None: + self.in_folder = Path(in_folder).resolve() + self.folder = self.in_folder.parent / (str(self.in_folder.name) + "_signed") + self.keys["root"] = [ + get_fingerprint(os.environ["KEY1"]), + get_fingerprint(os.environ["KEY2"]), + ] + self.keys = normalize_keys(self.keys) + + def make_signed_repo(self) -> Path: + print("[reposigner] Using keys:", self.keys) + print("[reposigner] Using folder:", self.folder) + + self.folder.mkdir(exist_ok=True) + self.create_root(self.keys) + self.create_key_mgr(self.keys) + for f in glob.glob(str(self.in_folder / "**" / "repodata.json")): + self.sign_repodata(Path(f), self.keys) + return self.folder + + def create_root(self, keys): + root_keys = keys["root"] + + root_pubkeys = [k["public"] for k in root_keys] + key_mgr_pubkeys = [k["public"] for k in keys["key_mgr"]] + + root_version = 1 + + root_md = cct_metadata_construction.build_root_metadata( + root_pubkeys=root_pubkeys[0:1], + root_threshold=1, + root_version=root_version, + key_mgr_pubkeys=key_mgr_pubkeys, + key_mgr_threshold=1, + ) + + # Wrap the metadata in a signing envelope. + root_md = cct_signing.wrap_as_signable(root_md) + + root_md_serialized_unsigned = cct_common.canonserialize(root_md) + + root_filepath = self.folder / f"{root_version}.root.json" + print("Writing out: ", root_filepath) + # Write unsigned sample root metadata. + with open(root_filepath, "wb") as fout: + fout.write(root_md_serialized_unsigned) + + # This overwrites the file with a signed version of the file. + cct_root_signing.sign_root_metadata_via_gpg( + root_filepath, root_keys[0]["fingerprint"] + ) + + # Load untrusted signed root metadata. + signed_root_md = cct_common.load_metadata_from_file(root_filepath) + + cct_authentication.verify_signable(signed_root_md, root_pubkeys, 1, gpg=True) + + print("[reposigner] Root metadata signed & verified!") + + def create_key_mgr(self, keys): + private_key_key_mgr = cct_common.PrivateKey.from_hex( + keys["key_mgr"][0]["private"] + ) + pkg_mgr_pub_keys = [k["public"] for k in keys["pkg_mgr"]] + key_mgr = cct_metadata_construction.build_delegating_metadata( + metadata_type="key_mgr", # 'root' or 'key_mgr' + delegations={"pkg_mgr": {"pubkeys": pkg_mgr_pub_keys, "threshold": 1}}, + version=1, + # timestamp default: now + # expiration default: now plus root expiration default duration + ) + + key_mgr = cct_signing.wrap_as_signable(key_mgr) + + # sign dictionary in place + cct_signing.sign_signable(key_mgr, private_key_key_mgr) + + key_mgr_serialized = cct_common.canonserialize(key_mgr) + with open(self.folder / "key_mgr.json", "wb") as fobj: + fobj.write(key_mgr_serialized) + + # let's run a verification + root_metadata = cct_common.load_metadata_from_file(self.folder / "1.root.json") + key_mgr_metadata = cct_common.load_metadata_from_file( + self.folder / "key_mgr.json" + ) + + cct_common.checkformat_signable(root_metadata) + + if "delegations" not in root_metadata["signed"]: + raise ValueError('Expected "delegations" entry in root metadata.') + + root_delegations = root_metadata["signed"]["delegations"] # for brevity + cct_common.checkformat_delegations(root_delegations) + if "key_mgr" not in root_delegations: + raise ValueError( + 'Missing expected delegation to "key_mgr" in root metadata.' + ) + cct_common.checkformat_delegation(root_delegations["key_mgr"]) + + # Doing delegation processing. + cct_authentication.verify_delegation("key_mgr", key_mgr_metadata, root_metadata) + + print("[reposigner] success: key mgr metadata verified based on root metadata.") + + return key_mgr + + def sign_repodata(self, repodata_fn, keys): + target_folder = self.folder / repodata_fn.parent.name + if not target_folder.exists(): + target_folder.mkdir() + + final_fn = target_folder / repodata_fn.name + print("copy", repodata_fn, final_fn) + shutil.copyfile(repodata_fn, final_fn) + + pkg_mgr_key = keys["pkg_mgr"][0]["private"] + cct_signing.sign_all_in_repodata(str(final_fn), pkg_mgr_key) + print(f"[reposigner] Signed {final_fn}") + + +class ChannelHandler(SimpleHTTPRequestHandler): + url_pattern = re.compile(r"^/(?:t/[^/]+/)?([^/]+)") + + def do_GET(self) -> None: + # First extract channel name + channel_name = None + if tuple(channels.keys()) != (None,): + match = self.url_pattern.match(self.path) + if match: + channel_name = match.group(1) + # Strip channel for file server + start, end = match.span(1) + self.path = self.path[:start] + self.path[end:] + + # Then dispatch to appropriate auth method + if channel_name in channels: + channel = channels[channel_name] + self.directory = channel["directory"] + auth = channel["auth"] + if auth == "none": + return SimpleHTTPRequestHandler.do_GET(self) + elif auth == "basic": + server_key = base64.b64encode( + bytes(f"{channel['user']}:{channel['password']}", "utf-8") + ).decode("ascii") + return self.basic_do_GET(server_key=server_key) + elif auth == "bearer": + return self.bearer_do_GET(server_key=channel["bearer"]) + elif auth == "token": + return self.token_do_GET(server_token=channel["token"]) + + self.send_response(404) + + def basic_do_HEAD(self) -> None: + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def basic_do_AUTHHEAD(self) -> None: + self.send_response(401) + self.send_header("WWW-Authenticate", 'Basic realm="Test"') + self.send_header("Content-type", "text/html") + self.end_headers() + + def bearer_do_GET(self, server_key: str) -> None: + auth_header = self.headers.get("Authorization", "") + print(auth_header) + print(f"Bearer {server_key}") + if not auth_header or auth_header != f"Bearer {server_key}": + self.send_response(403) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(b"no valid api key received") + else: + SimpleHTTPRequestHandler.do_GET(self) + + def basic_do_GET(self, server_key: str) -> None: + """Present frontpage with basic user authentication.""" + auth_header = self.headers.get("Authorization", "") + + if not auth_header: + self.basic_do_AUTHHEAD() + self.wfile.write(b"no auth header received") + elif auth_header == "Basic " + server_key: + SimpleHTTPRequestHandler.do_GET(self) + else: + self.basic_do_AUTHHEAD() + self.wfile.write(auth_header.encode("ascii")) + self.wfile.write(b"not authenticated") + + token_pattern = re.compile("^/t/([^/]+?)/") + + def token_do_GET(self, server_token: str) -> None: + """Present frontpage with user authentication.""" + match = self.token_pattern.search(self.path) + if match: + prefix_length = len(match.group(0)) - 1 + new_path = self.path[prefix_length:] + found_token = match.group(1) + if found_token == server_token: + self.path = new_path + return SimpleHTTPRequestHandler.do_GET(self) + + self.send_response(403) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(b"no valid api key received") + + +global_parser = argparse.ArgumentParser( + description="Start a multi-channel conda package server." +) +global_parser.add_argument("-p", "--port", type=int, default=8000, help="Port to use.") + +channel_parser = argparse.ArgumentParser( + description="Start a simple conda package server." +) +channel_parser.add_argument( + "-d", + "--directory", + type=str, + default=os.getcwd(), + help="Root directory for serving.", +) +channel_parser.add_argument( + "-n", + "--name", + type=str, + default=None, + help="Unique name of the channel used in URL", +) +channel_parser.add_argument( + "-a", + "--auth", + default=None, + type=str, + help="auth method (none, basic, token, or bearer)", +) +channel_parser.add_argument( + "--sign", + action="store_true", + help="Sign repodata (note: run generate_gpg_keys.sh before)", +) +channel_parser.add_argument( + "--token", + type=str, + default=None, + help="Use token as API Key", +) +channel_parser.add_argument( + "--bearer", + type=str, + default=None, + help="Use bearer token as API Key", +) +channel_parser.add_argument( + "--user", + type=str, + default=None, + help="Use token as API Key", +) +channel_parser.add_argument( + "--password", + type=str, + default=None, + help="Use token as API Key", +) + + +# Gobal args can be given anywhere with the first set of args for backward compatibility. +args, argv_remaining = global_parser.parse_known_args() +PORT = args.port + +# Iteratively parse arguments in sets. +# Each argument set, separated by -- in the CLI is for a channel. +# Credits: @hpaulj on SO https://stackoverflow.com/a/26271421 +channels = {} +while argv_remaining: + args, argv_remaining = channel_parser.parse_known_args(argv_remaining) + # Drop leading -- to move to next argument set + argv_remaining = argv_remaining[1:] + # Consolidation + if not args.auth: + if args.user and args.password: + args.auth = "basic" + elif args.token: + args.auth = "token" + elif args.bearer: + args.auth = "bearer" + else: + args.auth = "none" + if args.sign: + if not conda_content_trust_available: + fatal_error("Conda content trust not installed!") + args.directory = RepoSigner(args.directory).make_signed_repo() + + # name = args.name if args.name else Path(args.directory).name + # args.name = name + channels[args.name] = vars(args) + +print(channels) + +# Unamed channel in multi-channel case would clash URLs but we want to allow +# a single unamed channel for backward compatibility. +if (len(channels) > 1) and (None in channels): + fatal_error("Cannot use empty channel name when using multiple channels") + +server = HTTPServer(("", PORT), ChannelHandler) +print("Server started at localhost:" + str(PORT)) +try: + server.serve_forever() +except: + # Catch all sorts of interrupts + print("Shutting server down") + server.shutdown() + print("Server shut down")