add rattler_auth to authenticate requests

crates/rattler-bin/Cargo.toml

@@ -16,8 +16,8 @@ path = "src/main.rs"
 
 [features]
 default = ["native-tls"]
-native-tls = ["reqwest/native-tls", "rattler/native-tls", "rattler_repodata_gateway/native-tls"]
-rustls-tls = ["reqwest/rustls-tls", "rattler/rustls-tls", "rattler_repodata_gateway/rustls-tls"]
+native-tls = ["reqwest/native-tls", "rattler/native-tls", "rattler_repodata_gateway/native-tls", "rattler_networking/native-tls"]
+rustls-tls = ["reqwest/rustls-tls", "rattler/rustls-tls", "rattler_repodata_gateway/rustls-tls", "rattler_networking/rustls-tls"]
 
 [dependencies]
 anyhow = "1.0.71"
@@ -29,6 +29,7 @@ indicatif = "0.17.5"
 itertools = "0.10.5"
 once_cell = "1.18.0"
 rattler = { version = "0.2.0", path = "../rattler", default-features = false }
+rattler_networking = { version = "0.2.0", path = "../rattler_networking", default-features = false }
 rattler_conda_types = { version = "0.2.0", path = "../rattler_conda_types" }
 rattler_repodata_gateway = { version = "0.2.0", path = "../rattler_repodata_gateway", features = ["sparse"], default-features = false }
 rattler_solve = { version = "0.2.0", path = "../rattler_solve" }

crates/rattler-bin/src/commands/create.rs

@@ -10,6 +10,7 @@ use rattler_conda_types::{
     Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, Platform, PrefixRecord,
     RepoDataRecord,
 };
+use rattler_networking::{AuthenticatedClient, AuthenticationStorage};
 use rattler_repodata_gateway::fetch::{
     CacheResult, DownloadProgress, FetchRepoDataError, FetchRepoDataOptions,
 };
@@ -93,6 +94,14 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         .no_gzip()
         .build()
         .expect("failed to create client");
+
+    let auth_dir = dirs::config_local_dir()
+        .ok_or_else(|| anyhow::anyhow!("could not determine cache directory for current platform"))?
+        .join("rattler/auth");
+
+    let authentication_storage = AuthenticationStorage::new("rattler_credentials", &auth_dir);
+
+    let download_client = AuthenticatedClient::from_client(download_client, authentication_storage);
     let multi_progress = global_multi_progress();
 
     let repodata_cache_path = cache_dir.join("repodata");
@@ -197,7 +206,7 @@ async fn execute_transaction(
     transaction: Transaction<PrefixRecord, RepoDataRecord>,
     target_prefix: PathBuf,
     cache_dir: PathBuf,
-    download_client: Client,
+    download_client: AuthenticatedClient,
 ) -> anyhow::Result<()> {
     // Open the package cache
     let package_cache = PackageCache::new(cache_dir.join("pkgs"));
@@ -277,7 +286,7 @@ async fn execute_transaction(
 #[allow(clippy::too_many_arguments)]
 async fn execute_operation(
     target_prefix: &Path,
-    download_client: Client,
+    download_client: AuthenticatedClient,
     package_cache: &PackageCache,
     install_driver: &InstallDriver,
     download_pb: Option<&ProgressBar>,
@@ -463,7 +472,7 @@ async fn fetch_repo_data_records_with_progress(
     channel: Channel,
     platform: Platform,
     repodata_cache: &Path,
-    client: Client,
+    client: AuthenticatedClient,
     multi_progress: indicatif::MultiProgress,
 ) -> Result<Option<SparseRepoData>, anyhow::Error> {
     // Create a progress bar

crates/rattler/Cargo.toml

@@ -34,6 +34,7 @@ once_cell = "1.18.0"
 pin-project-lite = "0.2.9"
 rattler_conda_types = { version = "0.2.0", path = "../rattler_conda_types" }
 rattler_digest = { version = "0.2.0", path = "../rattler_digest" }
+rattler_networking = { version = "0.2.0", path = "../rattler_networking", default-features = false }
 rattler_package_streaming = { version = "0.2.0", path = "../rattler_package_streaming", features = ["reqwest", "tokio"], default-features = false }
 regex = "1.8.4"
 reqwest = { version = "0.11.18", default-features = false, features = ["stream", "json", "gzip"] }

crates/rattler/src/install/mod.rs

@@ -563,7 +563,8 @@ mod test {
     use rattler_conda_types::conda_lock::CondaLock;
     use rattler_conda_types::package::ArchiveIdentifier;
     use rattler_conda_types::{ExplicitEnvironmentSpec, Platform, Version};
-    use reqwest::Client;
+    use rattler_networking::AuthenticatedClient;
+
     use std::env::temp_dir;
     use std::process::Command;
     use std::str::FromStr;
@@ -618,7 +619,7 @@ mod test {
         let package_cache = PackageCache::new(temp_dir().join("rattler").join(cache_name));
 
         // Create an HTTP client we can use to download packages
-        let client = Client::new();
+        let client = AuthenticatedClient::default();
 
         // Specify python version
         let python_version =

crates/rattler/src/package_cache.rs

@@ -5,7 +5,7 @@ use fxhash::FxHashMap;
 use itertools::Itertools;
 use rattler_conda_types::package::ArchiveIdentifier;
 use rattler_conda_types::PackageRecord;
-use reqwest::Client;
+use rattler_networking::AuthenticatedClient;
 use std::error::Error;
 use std::{
     fmt::{Display, Formatter},
@@ -180,7 +180,7 @@ impl PackageCache {
         &self,
         pkg: impl Into<CacheKey>,
         url: Url,
-        client: Client,
+        client: AuthenticatedClient,
     ) -> Result<PathBuf, PackageCacheError> {
         self.get_or_fetch(pkg, move |destination| async move {
             tracing::debug!("downloading {} to {}", &url, destination.display());

crates/rattler_networking/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "rattler_networking"
+version.workspace = true
+edition.workspace = true
+authors = ["Wolf Vollprecht <w.vollprecht@gmail.com>"]
+description = "Authenticated requests in the conda ecosystem"
+categories.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+readme.workspace = true
+
+[features]
+native-tls = ['reqwest/native-tls']
+rustls-tls = ['reqwest/rustls-tls']
+
+[dependencies]
+anyhow = "1.0.71"
+dirs = "5.0.1"
+keyring = "2.0.2"
+lazy_static = "1.4.0"
+libc = "0.2.144"
+reqwest = { version = "0.11.17", features = ["blocking"], default-features = false}
+serde = "1.0.163"
+serde_json = "1.0.96"
+thiserror = "1.0.40"
+tracing = "0.1.37"
+
+[dev-dependencies]
+anyhow = "1.0.71"
+insta = { version = "1.29.0", features = ["json"] }
+tempfile = "3.5.0"

crates/rattler_networking/src/authentication_storage/authentication.rs

@@ -0,0 +1,40 @@
+//! Authentication methods for the conda ecosystem
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+/// The different Authentication methods that are supported in the conda
+/// ecosystem
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Debug)]
+pub enum Authentication {
+    /// A bearer token is sent as a header of the form
+    /// `Authorization: Bearer {TOKEN}`
+    BearerToken(String),
+    /// A basic authentication token is sent as HTTP basic auth
+    BasicHTTP {
+        /// The username to use for basic auth
+        username: String,
+        /// The password to use for basic auth
+        password: String,
+    },
+    /// A conda token is sent in the URL as `/t/{TOKEN}/...`
+    CondaToken(String),
+}
+
+/// An error that can occur when parsing an authentication string
+#[derive(Debug)]
+pub enum AuthenticationParseError {
+    /// The scheme is not valid
+    InvalidScheme,
+    /// The token could not be parsed
+    InvalidToken,
+}
+
+impl FromStr for Authentication {
+    type Err = AuthenticationParseError;
+
+    /// Parse an authentication string into an Authentication struct
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        serde_json::from_str(s).map_err(|_| AuthenticationParseError::InvalidToken)
+    }
+}

crates/rattler_networking/src/authentication_storage/fallback_storage.rs

@@ -0,0 +1,117 @@
+//! Fallback storage for passwords.
+use std::{
+    path::PathBuf,
+    sync::{Arc, Mutex},
+};
+
+/// A struct that implements storage and access of authentication
+/// information backed by a on-disk JSON file
+#[derive(Clone)]
+pub struct FallbackStorage {
+    /// The path to the JSON file
+    pub path: PathBuf,
+
+    /// A mutex to ensure that only one thread accesses the file at a time
+    mutex: Arc<Mutex<()>>,
+}
+
+/// An error that can occur when accessing the fallback storage
+#[derive(thiserror::Error, Debug)]
+pub enum FallbackStorageError {
+    /// An IO error occurred when accessing the fallback storage
+    #[error("IO error: {0}")]
+    IOError(#[from] std::io::Error),
+
+    /// An error occurred when (de)serializing the credentials
+    #[error("JSON error: {0}")]
+    JSONError(#[from] serde_json::Error),
+}
+
+impl FallbackStorage {
+    /// Create a new fallback storage with the given path
+    pub fn new(path: PathBuf) -> Self {
+        Self {
+            path,
+            mutex: Arc::new(Mutex::new(())),
+        }
+    }
+
+    /// Store the given authentication information for the given host
+    pub fn set_password(&self, host: &str, password: &str) -> Result<(), FallbackStorageError> {
+        let _lock = self.mutex.lock().unwrap();
+        let mut dict = self.read_json()?;
+        dict.insert(host.to_string(), password.to_string());
+        self.write_json(&dict)
+    }
+
+    /// Retrieve the authentication information for the given host
+    pub fn get_password(&self, host: &str) -> Result<Option<String>, FallbackStorageError> {
+        let _lock = self.mutex.lock().unwrap();
+        let dict = self.read_json()?;
+        Ok(dict.get(host).cloned())
+    }
+
+    /// Delete the authentication information for the given host
+    pub fn delete_password(&self, host: &str) -> Result<(), FallbackStorageError> {
+        let _lock = self.mutex.lock().unwrap();
+        let mut dict = self.read_json()?;
+        dict.remove(host);
+        self.write_json(&dict)
+    }
+
+    /// Read the JSON file and deserialize it into a HashMap, or return an empty HashMap if the file
+    /// does not exist
+    fn read_json(&self) -> Result<std::collections::HashMap<String, String>, FallbackStorageError> {
+        if !self.path.exists() {
+            return Ok(std::collections::HashMap::new());
+        }
+        let file = std::fs::File::open(&self.path)?;
+        let reader = std::io::BufReader::new(file);
+        let dict = serde_json::from_reader(reader)?;
+        Ok(dict)
+    }
+
+    /// Serialize the given HashMap and write it to the JSON file
+    fn write_json(
+        &self,
+        dict: &std::collections::HashMap<String, String>,
+    ) -> Result<(), FallbackStorageError> {
+        if !self.path.exists() {
+            std::fs::create_dir_all(self.path.parent().unwrap())?;
+        }
+        let file = std::fs::File::create(&self.path)?;
+        let writer = std::io::BufWriter::new(file);
+        serde_json::to_writer(writer, dict)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_fallback_storage() {
+        let file = tempdir().unwrap();
+        let path = file.path().join("test.json");
+
+        let storage = FallbackStorage::new(path.clone());
+
+        assert_eq!(storage.get_password("test").unwrap(), None);
+
+        storage.set_password("test", "password").unwrap();
+        assert_eq!(
+            storage.get_password("test").unwrap(),
+            Some("password".to_string())
+        );
+
+        storage.delete_password("test").unwrap();
+        assert_eq!(storage.get_password("test").unwrap(), None);
+
+        let mut file = std::fs::File::create(&path).unwrap();
+        file.write_all(b"invalid json").unwrap();
+        assert!(storage.get_password("test").is_err());
+    }
+}

crates/rattler_networking/src/authentication_storage/mod.rs

@@ -0,0 +1,4 @@
+//! This module provides a way to store and retrieve authentication information for a given host.
+pub mod authentication;
+pub mod fallback_storage;
+pub mod storage;