Merge branch 'github-auth'

Cargo.toml

@@ -16,7 +16,7 @@ clap_derive = "4.3.12"
 directories = "5.0.1"
 chardetng = "0.1.17"
 chrono = "0.4.26"
-dialoguer = "0.10.4"
+dialoguer = "0.11.0"
 encoding_rs = "0.8.32"
 env_logger = "0.10.0"
 filetime = "0.2.22"
@@ -28,15 +28,15 @@ handlebars = "4.3.7"
 humantime = "2.1.0"
 indicatif = "0.17.6"
 indoc ="2.0.3"
-itertools = "0.11.0"
+itertools = "0.12.0"
 lazy_static = "1.4.0"
 log = "0.4.19"
 patch = "0.7.0"
 path-clean = "1.0.1"
 path-absolutize = "3.1.0"
 percent-encoding = "2.3.0"
 regex = "1.9.3"
-reqwest = { version = "0.11.18", features = ["stream"] }
+reqwest = { version = "0.11.18", features = ["stream", "json"] }
 serde = { version = "1.0.183", features = ["derive"] }
 serde_json = "1.0.104"
 serde_path_to_error = "0.1.14"

README.md

@@ -315,6 +315,33 @@ All properties are optional.
 # can be an absolute path, or can use ~ exapansion on UNIX-like OSes
 archive_cache: ~/path/to/my/cache
 ```
+## Authenticated github downloads
+
+It is possible to download from a private repository.
+
+- create a _Personal Access Token_ , see https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic
+- create a `modda-credentials.yml` file in the configuration location (the same location as `modda.yml`` file)
+
+```yaml
+github:
+    personal_tokens:
+        my_repositories: XXXXX # copied from github
+```
+
+Then the `location` property of the mod needs to tell which token must be used.
+
+```yaml
+mods:
+  - name: MysteriousMod
+    components:
+      - 0
+    location:
+      github_user: Myself
+      repository: my_private_repo
+      release: V1.0
+      asset: my_private_asset.zip
+      auth: PAT my_repositories
+```
 
 ## Weidu
 

src/credentials.rs

@@ -0,0 +1,60 @@
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::BufReader;
+
+use anyhow::{Result, bail};
+use log::debug;
+use serde::Deserialize;
+
+use crate::progname::PROGNAME;
+
+#[derive(Deserialize, Debug, Default)]
+pub struct Credentials {
+    #[serde(default)]
+    pub github: Option<GithubCredentials>,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(untagged)]
+pub enum GithubCredentials {
+    PersonalAccessToken { personal_tokens: HashMap<String, String> },
+    // maybe oauth + bearer auth later, if possible
+}
+
+impl Credentials {
+    pub fn read() -> Result<Self> {
+        let yaml_name = format!("{prog_name}-credentials.yml", prog_name = PROGNAME);
+        let file = match File::open(&yaml_name) {
+            Ok(file) => Some(file),
+            Err(_error) => {
+                if let Some(proj_dir) = directories::ProjectDirs::from("", "", PROGNAME) {
+                    let conf_dir = proj_dir.config_dir();
+                    let conf_path = conf_dir.join(&yaml_name);
+                    debug!("Checking credentials file at {:?}", conf_path);
+                    match File::open(conf_path) {
+                        Ok(file) => {
+                            debug!("found credentials file");
+                            Some(file)
+                        },
+                        Err(_error) => None,
+                    }
+                } else {
+                    None
+                }
+            }
+        };
+
+        match file {
+            None => Ok(Self::default()),
+            Some(file) => {
+                let reader = BufReader::new(file);
+                let config: Self = match serde_yaml::from_reader(reader) {
+                    Err(error) => bail!("Invalid credentials file\n {error}"),
+                    Ok(config) => config,
+                };
+                Ok(config)
+            }
+        }
+    }
+}

src/download.rs

@@ -9,8 +9,10 @@ use filetime::FileTime;
 use futures_util::stream::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle, ProgressState};
 use log::{debug, info};
+use reqwest::header::{HeaderMap, USER_AGENT};
 
 use crate::module::refresh::RefreshCondition;
+use crate::progname::PROGNAME;
 
 
 #[cfg_attr(test, faux::create)]
@@ -28,7 +30,8 @@ impl Downloader {
         Downloader {}
     }
 
-    pub async fn download(&self, url: &str, dest_dir: &PathBuf, file_name: PathBuf, opts: &DownloadOpts)-> Result<PathBuf> {
+    pub async fn download(&self, url: &str, dest_dir: &PathBuf, file_name: PathBuf, opts: &DownloadOpts,
+                            headers: &Option<HeaderMap>) -> Result<PathBuf> {
         info!("obtaining {:?}, url is {} (cache={:?})", file_name, url, dest_dir);
 
         // check if archive exists in the cache
@@ -43,7 +46,7 @@ impl Downloader {
 
         let partial_name = get_partial_filename(&file_name)?;
 
-        if let Err(error) = self.download_partial(url, &partial_name, &dest_dir).await {
+        if let Err(error) = self.download_partial(url, &partial_name, &dest_dir, headers).await {
             bail!("download_partial failed for {} to {:?}\n  {}", url, partial_name, error);
         };
 
@@ -57,7 +60,8 @@ impl Downloader {
         }
     }
 
-    pub async fn download_partial(&self, url: &str, partial_name: &PathBuf, dest_dir: &PathBuf)  -> Result<()> {
+    pub async fn download_partial(&self, url: &str, partial_name: &PathBuf, dest_dir: &PathBuf,
+                                    headers: &Option<HeaderMap>)  -> Result<()> {
         info!("download {} to {:?}", url, dest_dir);
         std::fs::create_dir_all(dest_dir)?;
 
@@ -67,8 +71,14 @@ impl Downloader {
             Err(error) => bail!("failed to create file {:?}\n -> {:?}", partial_name, error),
             Ok(file) => file,
         };
+        let mut request = client.get(url)
+            .header(USER_AGENT, PROGNAME);
 
-        let response = match client.get(url).send().await {
+        if let Some(headers) = headers {
+            request = request.headers(headers.to_owned());
+        }
+
+        let response = match request.send().await {
             Ok(response) => response,
             Err(error) => bail!("HTTP download failed\n -> {:?}", error),
         };

src/get_module.rs

@@ -162,64 +162,6 @@ mod test_retrieve_location {
     use anyhow::bail;
     use faux::when;
 
-    /**
-     * Checks that for a github mod, retrieve_location(...) returns whetever is returned by download(...).
-     */
-    #[tokio::test]
-    async fn retrieve_github_location() {
-
-        let location = ConcreteLocation {
-            source: Source::Github(Github {
-                github_user: "username".to_string(),
-                repository: "repository".to_string(),
-                descriptor: Release {
-                    release: Some("V1".to_string()),
-                    asset: "repository_v1.zip".to_string(),
-                },
-                ..Default::default()
-            }),
-            ..ConcreteLocation::default()
-        };
-        let module = WeiduMod {
-            location: Some(Location::Concrete { concrete: location.clone() }),
-            ..WeiduMod::default()
-        };
-        let global = Global::default();
-        let global_locations = GlobalLocations::default();
-        let opts = Install::default();
-        let config = Config {
-            archive_cache: Some("/cache_path".to_string()),
-            extract_location: Some("/tmp".to_string()),
-            weidu_path: None,
-            extractors: HashMap::new(),
-        };
-
-        let expected_dest = PathBuf::from("/cache_path/github/username/repository");
-
-        let game_dir = CanonPath::new("some_dir").unwrap();
-        let cache = Cache::Path(PathBuf::from("/cache_path"));
-
-        let mut downloader = Downloader::faux();
-        when!(
-            downloader.download(_, {expected_dest}, _, _)
-        ).then(|(_, _, _, _)| Ok(PathBuf::from("cache_dir/directory/filename.zip")));
-        when!(
-            downloader.download_partial(_, _, _)
-        ).then(|(_, _, _)| bail!("Should not be called"));
-        when!(
-            downloader.rename_partial(_, _)
-        ).then(|(_, _)| bail!("Should not be called"));
-
-        let module_download: ModuleDownload<'_> = ModuleDownload::new(&config, &global, &global_locations, &opts,
-                                                                            &downloader, &game_dir, &cache);
-
-        let result = module_download.retrieve_location(&location, &module.name);
-        assert_eq!(
-            result.await.unwrap(),
-            PathBuf::from("cache_dir/directory/filename.zip")
-        )
-    }
-
     /**
      * Check http location.
      * Should be <cache_path>/http/<host_name>/<file_name>
@@ -255,11 +197,11 @@ mod test_retrieve_location {
 
         let mut downloader = Downloader::faux();
         when!(
-            downloader.download(_, {expected_dest}, _, _)
-        ).then(|(_, _, _, _)| Ok(PathBuf::from("/cache_path/http/example.com/some_mod.zip")));
+            downloader.download(_, {expected_dest}, _, _, _)
+        ).then(|(_, _, _, _, _)| Ok(PathBuf::from("/cache_path/http/example.com/some_mod.zip")));
         when!(
-            downloader.download_partial(_, _, _)
-        ).then(|(_, _, _)| bail!("Should not be called"));
+            downloader.download_partial(_, _, _, _)
+        ).then(|(_, _, _, _)| bail!("Should not be called"));
         when!(
             downloader.rename_partial(_, _)
         ).then(|(_, _)| bail!("Should not be called"));

src/main.rs

@@ -5,6 +5,7 @@ mod apply_patch;
 mod args;
 mod cache;
 mod canon_path;
+mod credentials;
 mod download;
 mod file_installer;
 mod get_module;